{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "86a4bd36",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于隐马尔科夫模型的序列标注监督学习的代码。这里以命名实体识别任务为例，所使用的数据是Books数据集。为简单起见，标签序列采用BIO格式。首先构建数据集和标签集合："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41673bbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'B-NAME': 1, 'I-ORG': 2, 'B-PRO': 3, 'B-EDU': 4, 'I-NAME': 5, 'B-LOC': 6, 'B-TITLE': 7, 'B-RACE': 8, 'I-TITLE': 9, 'I-RACE': 10, 'I-PRO': 11, 'B-CONT': 12, 'I-EDU': 13, 'I-LOC': 14, 'I-CONT': 15, 'B-ORG': 16, 'O': 0}\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "\n",
    "class NERDataset:\n",
    "    def __init__(self):\n",
    "        train_file, test_file, label_file = 'ner_train.jsonl',\\\n",
    "            'ner_test.jsonl', 'ner_labels.json'\n",
    "        \n",
    "        def read_file(file_name):\n",
    "            with open(file_name, 'r', encoding='utf-8') as fin:\n",
    "                json_list = list(fin)\n",
    "            data_split = []\n",
    "            for json_str in json_list:\n",
    "                raw = json.loads(json_str)\n",
    "                d = {'tokens': raw['tokens'], 'tags': raw['tags']}\n",
    "                data_split.append(d)\n",
    "            return data_split\n",
    "        \n",
    "        # 读取JSON文件，转化为Python对象\n",
    "        self.train_data, self.test_data = read_file(train_file),\\\n",
    "            read_file(test_file)\n",
    "        self.label2id = json.loads(open(label_file, 'r').read())\n",
    "        self.id2label = {}\n",
    "        for k, v in self.label2id.items():\n",
    "            self.id2label[v] = k\n",
    "        print(self.label2id)\n",
    "\n",
    "    # 建立词表，过滤低频词\n",
    "    def build_vocab(self, min_freq=3):\n",
    "        # 统计词频\n",
    "        frequency = defaultdict(int)\n",
    "        for data in self.train_data:\n",
    "            tokens = data['tokens']\n",
    "            for token in tokens:\n",
    "                frequency[token] += 1\n",
    "                \n",
    "        print(f'unique tokens = {len(frequency)}, '+\\\n",
    "              f'total counts = {sum(frequency.values())}, '+\\\n",
    "              f'max freq = {max(frequency.values())}, '+\\\n",
    "              f'min freq = {min(frequency.values())}')\n",
    "        \n",
    "        # 由于词与标签一一对应，不能随便删除字符，\n",
    "        # 因此加入<unk>用于替代未登录词，加入<pad>用于批量训练\n",
    "        self.token2id = {'<pad>': 0, '<unk>': 1}\n",
    "        self.id2token = {0: '<pad>', 1: '<unk>'}\n",
    "        total_count = 0\n",
    "        # 将高频词加入词表\n",
    "        for token, freq in sorted(frequency.items(),\\\n",
    "                key=lambda x: -x[1]):\n",
    "            if freq > min_freq:\n",
    "                self.token2id[token] = len(self.token2id)\n",
    "                self.id2token[len(self.id2token)] = token\n",
    "                total_count += freq\n",
    "            else:\n",
    "                break\n",
    "        print(f'min_freq = {min_freq}, '\n",
    "              f'remaining tokens = {len(self.token2id)}, '\n",
    "              f'in-vocab rate = {total_count / sum(frequency.values())}')\n",
    "\n",
    "    # 将文本输入转化为词表中对应的索引\n",
    "    def convert_tokens_to_ids(self):\n",
    "        for data_split in [self.train_data, self.test_data]:\n",
    "            for data in data_split:\n",
    "                data['token_ids'] = []\n",
    "                for token in data['tokens']:\n",
    "                    data['token_ids'].append(self.token2id.get(token, 1)) \n",
    "        \n",
    "dataset = NERDataset()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25650a7c",
   "metadata": {},
   "source": [
    "接下来建立词表，将词元转换为索引，并将数据转化为适合训练的格式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a3eb49e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique tokens = 2454, total counts = 182068, max freq = 5630, min freq = 1\n",
      "min_freq = 0, remaining tokens = 2456, in-vocab rate = 1.0\n",
      "[595, 510, 353, 263, 424, 225, 764, 637, 353, 65, 80, 47, 41, 74, 53, 41, 141, 23, 74, 7, 259, 27, 135, 47, 31, 74, 67, 25, 1605, 815, 1157, 225, 595, 1158, 738, 688, 353, 1025, 65, 234, 27, 135, 316, 41, 31, 7, 80, 41, 42, 31, 47, 27, 528, 41, 141, 27, 135, 67, 1005, 1606, 363, 19, 677, 294, 37, 678, 1756, 604, 873, 1006, 1005, 353, 33, 30, 9, 113, 5, 45, 11, 82, 60, 21, 26, 5, 186, 153, 100, 565, 873, 353, 424, 1993, 387, 346, 250, 13, 5, 102, 214, 25, 11, 82, 60, 21, 26, 5, 186, 2, 7, 459, 97, 89, 147, 140, 139, 69, 46, 21, 12, 2, 7, 964, 98, 240, 677, 59, 9, 103, 11, 82, 60, 335, 200, 76, 26, 6, 110, 20, 9, 15, 4, 122, 539, 241, 99, 621, 40, 200, 57, 82, 156, 25, 11, 82, 381, 371, 91, 202, 6, 52, 2, 7, 670, 100, 84, 204, 53, 120, 27, 42, 3, 56, 1159, 3, 329, 31, 265, 31, 3, 56, 394, 394, 3, 84, 357, 84, 7, 25, 397, 7, 41, 7, 74, 7, 135, 7, 31, 7, 87, 7, 259, 7, 31, 7, 74, 7, 41, 7, 131, 7, 28, 289, 385, 4]\n",
      "['阿', '里', '斯', '提', '德', '·', '波', '拉', '斯', '（', 'A', 'r', 'i', 's', 't', 'i', 'd', 'e', 's', ' ', 'B', 'o', 'u', 'r', 'a', 's', '）', '和', '卢', '卡', '雅', '·', '阿', '伊', '纳', '罗', '斯', '托', '（', 'L', 'o', 'u', 'k', 'i', 'a', ' ', 'A', 'i', 'n', 'a', 'r', 'o', 'z', 'i', 'd', 'o', 'u', '）', '夫', '妇', '二', '人', '均', '拥', '有', '希', '腊', '比', '雷', '埃', '夫', '斯', '技', '术', '教', '育', '学', '院', '计', '算', '机', '工', '程', '学', '位', '以', '及', '色', '雷', '斯', '德', '谟', '克', '利', '特', '大', '学', '电', '子', '和', '计', '算', '机', '工', '程', '学', '位', '，', ' ', '都', '从', '事', '过', '软', '件', '开', '发', '工', '作', '，', ' ', '且', '目', '前', '均', '为', '教', '授', '计', '算', '机', '相', '关', '课', '程', '的', '高', '中', '教', '师', '。', '他', '们', '写', '了', '很', '多', '关', '于', '算', '法', '和', '计', '算', '思', '维', '方', '面', '的', '书', '，', ' ', '涉', '及', 'P', 'y', 't', 'h', 'o', 'n', '、', 'C', '#', '、', 'J', 'a', 'v', 'a', '、', 'C', '+', '+', '、', 'P', 'H', 'P', ' ', '和', 'V', ' ', 'i', ' ', 's', ' ', 'u', ' ', 'a', ' ', 'l', ' ', 'B', ' ', 'a', ' ', 's', ' ', 'i', ' ', 'c', ' ', '等', '语', '言', '。']\n"
     ]
    }
   ],
   "source": [
    "# 截取一部分数据便于更快训练\n",
    "dataset.train_data = dataset.train_data[:1000]\n",
    "dataset.test_data = dataset.test_data[:200]\n",
    "\n",
    "dataset.build_vocab(min_freq=0)\n",
    "dataset.convert_tokens_to_ids()\n",
    "print(dataset.train_data[0]['token_ids'])\n",
    "print([dataset.id2token[token_id] for token_id in \\\n",
    "       dataset.train_data[0]['token_ids']])\n",
    "\n",
    "def collect_data(data_split):\n",
    "    X, Y = [], []\n",
    "    for data in data_split:\n",
    "        assert len(data['token_ids']) == len(data['tags'])\n",
    "        X.append(data['token_ids'])\n",
    "        Y.append(data['tags'])\n",
    "    return X, Y\n",
    "\n",
    "train_X, train_Y = collect_data(dataset.train_data)\n",
    "test_X, test_Y = collect_data(dataset.test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7de17192",
   "metadata": {},
   "source": [
    "随后实现隐马尔科夫模型，使用最大似然估计得到模型参数，使用维特比算法进行解码。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3229fd6a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class HMM:\n",
    "    def __init__(self, n_tags, n_tokens):\n",
    "        self.n_tags = n_tags\n",
    "        self.n_tokens = n_tokens\n",
    "    \n",
    "    # 使用最大似然估计计算模型参数\n",
    "    def fit(self, X, Y):\n",
    "        Y0_cnt = np.zeros(self.n_tags)\n",
    "        YY_cnt = np.zeros((self.n_tags, self.n_tags))\n",
    "        YX_cnt = np.zeros((self.n_tags, self.n_tokens))\n",
    "        for x, y in zip(X, Y):\n",
    "            Y0_cnt[y[0]] += 1\n",
    "            last_y = y[0]\n",
    "            for i in range(1, len(y)):\n",
    "                YY_cnt[last_y, y[i]] += 1\n",
    "                last_y = y[i]\n",
    "            for xi, yi in zip(x, y):\n",
    "                YX_cnt[yi, xi] += 1\n",
    "        self.init_prob = Y0_cnt / Y0_cnt.sum()\n",
    "        self.transition_prob = YY_cnt\n",
    "        self.emission_prob = YX_cnt\n",
    "        for i in range(self.n_tags):\n",
    "            # 为了避免训练集过小时除0\n",
    "            yy_sum = YY_cnt[i].sum()\n",
    "            if yy_sum > 0:\n",
    "                self.transition_prob[i] = YY_cnt[i] / yy_sum\n",
    "            yx_sum = YX_cnt[i].sum()\n",
    "            if yx_sum > 0:\n",
    "                self.emission_prob[i] = YX_cnt[i] / yx_sum\n",
    "    \n",
    "    # 已知模型参数的条件下，使用维特比算法解码得到最优标签序列\n",
    "    def viterbi(self, x):\n",
    "        assert hasattr(self, 'init_prob') and hasattr(self,\\\n",
    "            'transition_prob') and hasattr(self, 'emission_prob')\n",
    "        Pi = np.zeros((len(x), self.n_tags))\n",
    "        Y = np.zeros((len(x), self.n_tags), dtype=np.int32)\n",
    "        # 初始化\n",
    "        for i in range(self.n_tags):\n",
    "            Pi[0, i] = self.init_prob[i] * self.emission_prob[i, x[0]]\n",
    "            Y[0, i] = -1\n",
    "        for t in range(1, len(x)):\n",
    "            for i in range(self.n_tags):\n",
    "                tmp = []\n",
    "                for j in range(self.n_tags):\n",
    "                    tmp.append(self.transition_prob[j, i] * Pi[t-1, j])\n",
    "                best_j = np.argmax(tmp)\n",
    "                # 维特比算法递推公式\n",
    "                Pi[t, i] = self.emission_prob[i, x[t]] * tmp[best_j]\n",
    "                Y[t, i] = best_j\n",
    "        y = [np.argmax(Pi[-1])]\n",
    "        for t in range(len(x)-1, 0, -1):\n",
    "            y.append(Y[t, y[-1]])\n",
    "        return np.max(Pi[len(x)-1]), y[::-1]\n",
    "    \n",
    "    def decode(self, X):\n",
    "        Y = []\n",
    "        for x in X:\n",
    "            _, y = self.viterbi(x)\n",
    "            Y.append(y)\n",
    "        return Y\n",
    "\n",
    "hmm = HMM(len(dataset.label2id), len(dataset.token2id))\n",
    "hmm.fit(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32868ed6",
   "metadata": {},
   "source": [
    "最后验证模型效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4f4b7e88",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.45762149610217284, recall = 0.3289222699093944, f1 = 0.38274259554692375\n",
      "precision = 0.4189636163175303, recall = 0.23270055113288426, f1 = 0.2992125984251969\n"
     ]
    }
   ],
   "source": [
    "def extract_entity(labels):\n",
    "    entity_list = []\n",
    "    entity_start = -1\n",
    "    entity_length = 0\n",
    "    entity_type = None\n",
    "    for token_index, label in enumerate(labels):\n",
    "        if label.startswith('B'):\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的B，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 记录新实体\n",
    "            entity_start = token_index\n",
    "            entity_length = 1\n",
    "            entity_type = label.split('-')[1]\n",
    "        elif label.startswith('I'):\n",
    "            if entity_start != -1:\n",
    "                # 上一个实体未关闭，遇到了一个新的I\n",
    "                if entity_type == label.split('-')[1]:\n",
    "                    # 若上一个实体与当前类型相同，长度+1\n",
    "                    entity_length += 1\n",
    "                else:\n",
    "                    # 若上一个实体与当前类型不同，\n",
    "                    # 将上一个实体加入列表，重置实体\n",
    "                    entity_list.append((entity_start, entity_length,\\\n",
    "                        entity_type))\n",
    "                    entity_start = -1\n",
    "                    entity_length = 0\n",
    "                    entity_type = None\n",
    "        else:\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的O，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 重置实体\n",
    "            entity_start = -1\n",
    "            entity_length = 0\n",
    "            entity_type = None\n",
    "    if entity_start != -1:\n",
    "        # 将上一个实体加入列表\n",
    "        entity_list.append((entity_start, entity_length, entity_type))\n",
    "    return entity_list\n",
    "\n",
    "def compute_metric(Y, P):\n",
    "    true_entity_set = set()\n",
    "    pred_entity_set = set()\n",
    "    for sent_no, labels in enumerate(Y):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            true_entity_set.add((sent_no, ent))\n",
    "    for sent_no, labels in enumerate(P):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            pred_entity_set.add((sent_no, ent))\n",
    "    if len(true_entity_set) > 0:\n",
    "        recall = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(true_entity_set)\n",
    "    else:\n",
    "        recall = 0\n",
    "    if len(pred_entity_set) > 0:\n",
    "        precision = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(pred_entity_set)\n",
    "    else:\n",
    "        precision = 0\n",
    "    if precision > 0 and recall > 0:\n",
    "        f1 = 2 * precision * recall / (precision + recall)\n",
    "    else:\n",
    "        f1 = 0\n",
    "    return precision, recall, f1\n",
    "\n",
    "train_P = hmm.decode(train_X)\n",
    "p, r, f = compute_metric(train_Y, train_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "test_P = hmm.decode(test_X)\n",
    "p, r, f = compute_metric(test_Y, test_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3abe8397",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于条件随机场的序列标注模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "787f8ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "代码修改自GitHub项目kmkurn/pytorch-crf\n",
    "（Copyright (c) 2019, Kemal Kurniawan, MIT License（见附录））\n",
    "\"\"\"\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "class CRFLayer(nn.Module):\n",
    "    def __init__(self, n_tags, n_features):\n",
    "        super().__init__()\n",
    "        self.n_tags = n_tags\n",
    "        self.n_features = n_features\n",
    "        # 定义模型参数\n",
    "        self.transitions = nn.Parameter(torch.empty(\\\n",
    "            n_tags, n_tags))\n",
    "        self.emission_weight = nn.Parameter(torch.empty(\\\n",
    "            n_features, n_tags))\n",
    "        self.start_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.end_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.reset_parameters()\n",
    "        \n",
    "    # 使用（-0.1,0.1）之间的均匀分布初始化参数\n",
    "    def reset_parameters(self):\n",
    "        nn.init.uniform_(self.transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.emission_weight, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.start_transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.end_transitions, -0.1, 0.1)\n",
    "    \n",
    "    # 使用动态规划计算得分\n",
    "    def compute_score(self, emissions, tags, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions[tags[0]] + \\\n",
    "            emissions[0, torch.arange(batch_size), tags[0]]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            score += self.transitions[tags[i-1], tags[i]] * masks[i]\n",
    "            score += emissions[i, torch.arange(batch_size),\\\n",
    "                tags[i]] * masks[i]\n",
    "        \n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        last_tags = tags[seq_ends, torch.arange(batch_size)]\n",
    "        score += self.end_transitions[last_tags]\n",
    "        return score\n",
    "    \n",
    "    # 计算配分函数\n",
    "    def computer_normalizer(self, emissions, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        # batch_size * n_tags, [起始分数 + y_0为某标签的发射分数 ...]\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            # batch_size * n_tags * 1 [y_{i-1}为某tag的总分]\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            # batch_size * 1 * n_tags [y_i为某标签的发射分数]\n",
    "            broadcast_emissions = emissions[i].unsqueeze(1)\n",
    "            # batch_size * n_tags * n_tags [任意y_{i-1}到y_i的总分]\n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emissions\n",
    "            # batch_size * n_tags [对y_{i-1}求和]\n",
    "            next_score = torch.logsumexp(next_score, dim=1)\n",
    "            # masks为True则更新，否则保留\n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score, score)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        return torch.logsumexp(score, dim=1)\n",
    "    \n",
    "    def forward(self, features, tags, masks):\n",
    "        \"\"\"\n",
    "        features: seq_len * batch_size * n_features\n",
    "        tags/masks: seq_len * batch_size\n",
    "        \"\"\"\n",
    "        _, batch_size, _ = features.size()\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        score = self.compute_score(emissions, tags, masks)\n",
    "        partition = self.computer_normalizer(emissions, masks)\n",
    "        \n",
    "        likelihood = score - partition\n",
    "        return likelihood.sum() / batch_size\n",
    "    \n",
    "    def decode(self, features, masks):\n",
    "        # 与computer_normalizer类似，sum变为max\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        history = []\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            broadcast_emission = emissions[i].unsqueeze(1)\n",
    "            \n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emission\n",
    "            next_score, indices = next_score.max(dim=1)\n",
    "            \n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score,\\\n",
    "                score)\n",
    "            history.append(indices)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        best_tags_list = []\n",
    "        \n",
    "        for idx in range(batch_size):\n",
    "            _, best_last_tag = score[idx].max(dim=0)\n",
    "            best_tags = [best_last_tag.item()]\n",
    "            \n",
    "            for hist in reversed(history[:seq_ends[idx]]):\n",
    "                best_last_tag = hist[idx][best_tags[-1]]\n",
    "                best_tags.append(best_last_tag.item())\n",
    "                \n",
    "            best_tags.reverse()\n",
    "            best_tags_list.append(best_tags)\n",
    "            \n",
    "        return best_tags_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "69f4af96",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size)\n",
    "        \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(embed, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        # 调用CRFLayer进行解码\n",
    "        return self.crf.decode(embed, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "178948b4",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, loss=53.11: 100%|█| 20/20 [02:26<00:00,  7.35s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGwCAYAAABIC3rIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABV7UlEQVR4nO3dd3xUVf7/8dek90nPEAgQINKbgUUQRZdmoVh2UUDU3+KqK4qxoOtXd1fdFRR3QQXLWr7q6rL43RVYu4AFQXqvSgskQEIghElvM/f3R8jVoYaQZErez8djHg/m3jMzn0Mk8/acc8+1GIZhICIiItKM+bm7ABERERF3UyASERGRZk+BSERERJo9BSIRERFp9hSIREREpNlTIBIREZFmT4FIREREmr0AdxfgLZxOJ4cOHSIyMhKLxeLuckRERKQODMOgqKiI5ORk/PzOPA6kQFRHhw4dIiUlxd1liIiISD1kZ2fTqlWrM55XIKqjyMhIoOYvNCoqys3ViIiISF0UFhaSkpJifo+fiQJRHdVOk0VFRSkQiYiIeJlzLXfRomoRERFp9hSIREREpNlTIBIREZFmT2uIREREAIfDQVVVlbvLkPMUGBiIv7//Bb+PApGIiDRrhmGQm5vL8ePH3V2K1FN0dDQ2m+2C9glUIBIRkWatNgwlJiYSFhamzXe9iGEYlJaWkpeXB0CLFi3q/V4KRCIi0mw5HA4zDMXFxbm7HKmH0NBQAPLy8khMTKz39JkWVYuISLNVu2YoLCzMzZXIhaj9+V3IGjAFIhERafY0TebdGuLnp0AkIiIizZ4CkYiIiDR7CkQiIiJC27ZteeGFF9z+Hu6iq8zcrKzSQY69jPjIYKJCAt1djoiIeIkrrriCXr16NVgAWbNmDeHh4Q3yXt5II0RuNv7Nlfzyb0v4ftdRd5ciIiI+xjAMqqur69Q2ISGhWV9tp0DkZi2sNfsn5NjL3VyJiIjAic3+Kqvd8jAMo0413n777SxZsoQXX3wRi8WCxWJh3759fPvtt1gsFr788kv69OlDcHAwS5cuZc+ePYwePZqkpCQiIiLo27cvixcvdnnPk6e7LBYLb775Jtdffz1hYWGkpaXx0UcfndffZVZWFqNHjyYiIoKoqCjGjBnD4cOHzfObNm3iyiuvJDIykqioKNLT01m7di0A+/fvZ+TIkcTExBAeHk7Xrl357LPPzuvzz4emzNzMZg0BIMde5uZKREQEoKzKQZc/fumWz97+9HDCgs791fziiy+yc+dOunXrxtNPPw3UjPDs27cPgEceeYS//vWvtGvXjujoaA4cOMA111zDX/7yF0JCQnj33XcZOXIkP/74I61btz7j5zz11FNMnz6d559/nlmzZjF+/Hj2799PbGzsOWs0DIPrrruO8PBwlixZQnV1Nffccw833XQT3377LQDjx4+nd+/evPrqq/j7+7Nx40YCA2uWj0yaNInKykq+++47wsPD2b59OxEREef83PpSIHKzFmYg0giRiIjUjdVqJSgoiLCwMGw22ynnn376aYYOHWo+j4uLo2fPnubzv/zlL8yfP5+PPvqIe++994yfc/vttzN27FgApk6dyqxZs1i9ejVXXXXVOWtcvHgxmzdvJjMzk5SUFADee+89unbtypo1a+jbty9ZWVlMmTKFTp06AZCWlma+PisrixtvvJHu3bsD0K5du3N+5oVQIHKz2hGiXAUiERGPEBroz/anh7vtsxtCnz59XJ6XlJTw1FNP8cknn3Do0CGqq6spKysjKyvrrO/To0cP88/h4eFERkaa9w07lx07dpCSkmKGIYAuXboQHR3Njh076Nu3Lw8++CB33HEH7733HkOGDOHXv/417du3B2Dy5Mn87ne/Y+HChQwZMoQbb7zRpZ6GpjVEbqY1RCIinsVisRAWFOCWR0PtmH3y1WJTpkzhww8/5JlnnmHp0qVs3LiR7t27U1lZedb3qZ2++vnfjdPprFMNhmGctj8/P/7kk0+ybds2rr32Wr7++mu6dOnC/PnzAbjjjjvYu3cvEyZMYMuWLfTp04dZs2bV6bPrQ4HIzWqnzA4XluN01m0xnYiISFBQEA6Ho05tly5dyu233871119P9+7dsdls5nqjxtKlSxeysrLIzs42j23fvh273U7nzp3NYxdddBEPPPAACxcu5IYbbuDtt982z6WkpHD33Xczb948HnroId54441Gq1eByM0SIoPxs0C10+BocYW7yxERES/Rtm1bVq1axb59+zh69OhZR246dOjAvHnz2LhxI5s2bWLcuHF1HumpryFDhtCjRw/Gjx/P+vXrWb16NbfeeiuDBg2iT58+lJWVce+99/Ltt9+yf/9+vv/+e9asWWOGpYyMDL788ksyMzNZv349X3/9tUuQamgKRG4W6O9HQmQwoGkzERGpu4cffhh/f3+6dOlCQkLCWdcDzZw5k5iYGAYMGMDIkSMZPnw4F198caPWZ7FYWLBgATExMVx++eUMGTKEdu3a8cEHHwDg7+9Pfn4+t956KxdddBFjxozh6quv5qmnngLA4XAwadIkOnfuzFVXXUXHjh155ZVXGq9eo66bHjRzhYWFWK1W7HY7UVFRDfre1738PRuzj/PaLelc1e3UqwVERKRxlJeXk5mZSWpqKiEhIe4uR+rpbD/Hun5/a4TIA7QwrzTTXkQiIiLuoEDkAWzai0hERMStFIg8gDZnFBERcS8FIg9QuxeRNmcUEXEPLaf1bg3x81Mg8gDmCFGh1hCJiDSl2o0HS0tL3VyJXIjan9/JG0meD926wwPUriE6bK/A6TTw82uYnUpFROTs/P39iY6ONm9HERYW1mC7RUvjMwyD0tJS8vLyiI6Oxt+//rc+USDyAElRIVgsUOlwkl9Sae5LJCIija/25qh1vUeXeJ7o6OjT3uT2fCgQeYBAfz8SIoLJK6og116uQCQi0oQsFgstWrQgMTGRqqoqd5cj5ykwMPCCRoZqKRB5iBbWEPKKKsixl9G9ldXd5YiINDv+/v4N8sUq3kmLqj1E7Tqi3EJdaSYiItLUFIg8RO2l94eOKxCJiIg0NQUiD6Hbd4iIiLiPApGH0O07RERE3EeByEOYu1VrDZGIiEiTUyDyED+/n5m2kBcREWlaCkQeIimqJhBVVjs5VlLp5mpERESaFwUiDxEU4Ed8RM2GjFpHJCIi0rQUiDxICy2sFhERcQsFIg+SFFUzQnSkqMLNlYiIiDQvCkQepHbK7GixApGIiEhTUiDyIApEIiIi7qFA5EHiIoIABSIREZGmpkDkQcwRoiJddi8iItKUFIg8iBmISjRCJCIi0pQUiDxIQuSJKTNdZSYiItKkFIg8SFx4zQhRYXk1FdUON1cjIiLSfCgQeRBraCABfhYA8ou1jkhERKSpKBB5ED8/i640ExERcQO3BqLvvvuOkSNHkpycjMViYcGCBS7nDcPgySefJDk5mdDQUK644gq2bdvm0qaiooL77ruP+Ph4wsPDGTVqFAcOHHBpU1BQwIQJE7BarVitViZMmMDx48cbuXf1U7uwWiNEIiIiTcetgaikpISePXsye/bs056fPn06M2bMYPbs2axZswabzcbQoUMpKioy22RkZDB//nzmzp3LsmXLKC4uZsSIETgcP63BGTduHBs3buSLL77giy++YOPGjUyYMKHR+1cftYHoiEaIREREmo7hIQBj/vz55nOn02nYbDbj2WefNY+Vl5cbVqvVeO211wzDMIzjx48bgYGBxty5c802Bw8eNPz8/IwvvvjCMAzD2L59uwEYK1euNNusWLHCAIwffvihzvXZ7XYDMOx2e327WCcPfLDBaPPoJ8bL3+xq1M8RERFpDur6/e2xa4gyMzPJzc1l2LBh5rHg4GAGDRrE8uXLAVi3bh1VVVUubZKTk+nWrZvZZsWKFVitVvr162e2ueSSS7BarWab06moqKCwsNDl0RQStDmjiIhIk/PYQJSbmwtAUlKSy/GkpCTzXG5uLkFBQcTExJy1TWJi4invn5iYaLY5nWnTpplrjqxWKykpKRfUn7rS/cxERESanscGoloWi8XluWEYpxw72cltTtf+XO/z2GOPYbfbzUd2dvZ5Vl4/8Sc2Z8zXbtUiIiJNxmMDkc1mAzhlFCcvL88cNbLZbFRWVlJQUHDWNocPHz7l/Y8cOXLK6NPPBQcHExUV5fJoCrqfmYiISNPz2ECUmpqKzWZj0aJF5rHKykqWLFnCgAEDAEhPTycwMNClTU5ODlu3bjXb9O/fH7vdzurVq802q1atwm63m208Se1u1ZoyExERaToB7vzw4uJidu/ebT7PzMxk48aNxMbG0rp1azIyMpg6dSppaWmkpaUxdepUwsLCGDduHABWq5WJEyfy0EMPERcXR2xsLA8//DDdu3dnyJAhAHTu3JmrrrqK3/72t/z9738H4M4772TEiBF07Nix6Tt9DrVTZsdKK6l2OAnw99jMKiIi4jPcGojWrl3LlVdeaT5/8MEHAbjtttt45513eOSRRygrK+Oee+6hoKCAfv36sXDhQiIjI83XzJw5k4CAAMaMGUNZWRmDBw/mnXfewd/f32zzz3/+k8mTJ5tXo40aNeqMex+5W2xYEBYLGEZNKEqMDHF3SSIiIj7PYhiG4e4ivEFhYSFWqxW73d7o64nS/7yI/JJKPr//Mjq3aJq1SyIiIr6ort/fmo/xQLr0XkREpGkpEHkg3eBVRESkaSkQeSBdei8iItK0FIg8kBmItDmjiIhIk1Ag8kC1l95rhEhERKRpKBB5IC2qFhERaVoKRB4oXouqRUREmpQCkQfSCJGIiEjTUiDyQLWBKL+4Eu2bKSIi0vgUiDxQ7T5E1U4De1mVm6sRERHxfQpEHig4wJ/IkJrbzGnaTEREpPEpEHmohBPTZkd06b2IiEijUyDyUOY6Im3OKCIi0ugUiDxUdFggAMdLtYZIRESksSkQeShraE0g0qJqERGRxqdA5KFqR4gUiERERBqfApGHig6rufT+eKkWVYuIiDQ2BSIPFRWqNUQiIiJNRYHIQ0VrDZGIiEiTUSDyUFpDJCIi0nQUiDyUVVNmIiIiTUaByENFh9YsqtYIkYiISONTIPJQ1hNTZmVVDiqqHW6uRkRExLcpEHmoyOAALJaaP2uUSEREpHEpEHkoPz/LT7tVax2RiIhIo1Ig8mC1l94f1wiRiIhIo1Ig8mAaIRIREWkaCkQezFp7+w6NEImIiDQqBSIPZk6Z6X5mIiIijUqByIPVTpkVaoRIRESkUSkQebDa23doykxERKRxKRB5MN2+Q0REpGkoEHkwq+54LyIi0iQUiDxYtK4yExERaRIKRB6sdg2RXVeZiYiINCoFIg+mKTMREZGmoUDkwaJ/FoicTsPN1YiIiPguBSIPFnUiEDkNKKqodnM1IiIivkuByIOFBPoTEljzI9LmjCIiIo1HgcjDRYeeuNJMexGJiIg0GgUiD/fTbtW60kxERKSxKBB5uChdaSYiItLoFIg8XLRu3yEiItLoFIg8nLk5o0aIREREGo0CkYfT5owiIiKNT4HIw5n3M9PtO0RERBqNApGHs2oNkYiISKNTIPJwmjITERFpfApEHk6LqkVERBqfApGH007VIiIijU+ByMNpykxERKTxeXQgqq6u5oknniA1NZXQ0FDatWvH008/jdPpNNsYhsGTTz5JcnIyoaGhXHHFFWzbts3lfSoqKrjvvvuIj48nPDycUaNGceDAgabuTr1YT0yZlVU5KK9yuLkaERER3+TRgei5557jtddeY/bs2ezYsYPp06fz/PPPM2vWLLPN9OnTmTFjBrNnz2bNmjXYbDaGDh1KUVGR2SYjI4P58+czd+5cli1bRnFxMSNGjMDh8PyAERkcgJ+l5s+6472IiEjjCHB3AWezYsUKRo8ezbXXXgtA27Zt+de//sXatWuBmtGhF154gccff5wbbrgBgHfffZekpCTmzJnDXXfdhd1u56233uK9995jyJAhALz//vukpKSwePFihg8fftrPrqiooKKiwnxeWFjYmF09Iz8/C1GhgRwvrcJeVkViVIhb6hAREfFlHj1CNHDgQL766it27twJwKZNm1i2bBnXXHMNAJmZmeTm5jJs2DDzNcHBwQwaNIjly5cDsG7dOqqqqlzaJCcn061bN7PN6UybNg2r1Wo+UlJSGqOLdWLez0wjRCIiIo3Co0eIHn30Uex2O506dcLf3x+Hw8EzzzzD2LFjAcjNzQUgKSnJ5XVJSUns37/fbBMUFERMTMwpbWpffzqPPfYYDz74oPm8sLDQbaHIGhYE+aUUlGi3ahERkcbg0YHogw8+4P3332fOnDl07dqVjRs3kpGRQXJyMrfddpvZzmKxuLzOMIxTjp3sXG2Cg4MJDg6+sA40kNgTC6sLdPsOERGRRuHRgWjKlCn8/ve/5+abbwage/fu7N+/n2nTpnHbbbdhs9mAmlGgFi1amK/Ly8szR41sNhuVlZUUFBS4jBLl5eUxYMCAJuxN/cVH1ASzo8UKRCIiIo3Bo9cQlZaW4ufnWqK/v7952X1qaio2m41FixaZ5ysrK1myZIkZdtLT0wkMDHRpk5OTw9atW70mEMWdCERHiirO0VJERETqw6NHiEaOHMkzzzxD69at6dq1Kxs2bGDGjBn85je/AWqmyjIyMpg6dSppaWmkpaUxdepUwsLCGDduHABWq5WJEyfy0EMPERcXR2xsLA8//DDdu3c3rzrzdPERNbtV52sNkYiISKPw6EA0a9Ys/vCHP3DPPfeQl5dHcnIyd911F3/84x/NNo888ghlZWXcc889FBQU0K9fPxYuXEhkZKTZZubMmQQEBDBmzBjKysoYPHgw77zzDv7+/u7o1nlLiDwxZaYRIhERkUZhMQzDcHcR3qCwsBCr1YrdbicqKqpJP3vZrqPc8tYq0hIjWPTgoCb9bBEREW9W1+9vj15DJDXiIzVlJiIi0pgUiLxA7VVmBaWVVDuc52gtIiIi50uByAvEhAXhZwHDgGPai0hERKTBKRB5AX8/C7HhNdNmR4sUiERERBqaApGX+GlzRl1pJiIi0tAUiLxEnLkXkQKRiIhIQ1Mg8hLmCJGmzERERBqcApGXiAvXlJmIiEhjUSDyErV7EekGryIiIg1PgchLaFG1iIhI41Eg8hK1N3hVIBIREWl4CkReonaEKF9TZiIiIg1OgchLmIGopALdj1dERKRhKRB5idqdqqscBoVl1W6uRkRExLcoEHmJkEB/IkMCADiidUQiIiINSoHIiyToSjMREZFGoUDkRczbd2hhtYiISINSIPIi2otIRESkcSgQeREFIhERkcahQORF4iJ0+w4REZHGoEDkRTRCJCIi0jgUiLyIbt8hIiLSOBSIvIhu3yEiItI4FIi8iKbMREREGocCkRepXVRdWumgtFK37xAREWkoCkReJCI4gOCAmh+Zps1EREQajgKRF7FYLOa0me5nJiIi0nAUiLyMeaVZkQKRiIhIQ1Eg8jIJkSGARohEREQakgKRl0mMqpkyyytUIBIREWkoCkReJjHyRCDSlJmIiEiDUSDyMgknAtGRonI3VyIiIuI7FIi8TOKJNUQaIRIREWk4CkRexpwy0xoiERGRBqNA5GVqF1UfLa7A6TTcXI2IiIhvUCDyMvERwVgsUO00OFaq3apFREQaggKRlwn09yM2rGZzxiNaRyQiItIgFIi8UIIuvRcREWlQCkReKDHqxJVmhbr0XkREpCEoEHkhbc4oIiLSsBSIvNBPmzMqEImIiDQEBSIv9NMIkabMREREGoICkRcyd6vW5owiIiINQoHIC9VuznikWIFIRESkISgQeaGf377DMLRbtYiIyIVSIPJCtVNmZVUOiiuq3VyNiIiI91Mg8kKhQf5EBgcAuvReRESkISgQeamEKN31XkREpKEoEHkpXXovIiLScOoViN59910+/fRT8/kjjzxCdHQ0AwYMYP/+/Q1WnJxZwol1RNqcUURE5MLVKxBNnTqV0NBQAFasWMHs2bOZPn068fHxPPDAAw1aoJyebt8hIiLScOoViLKzs+nQoQMACxYs4Fe/+hV33nkn06ZNY+nSpQ1a4MGDB7nllluIi4sjLCyMXr16sW7dOvO8YRg8+eSTJCcnExoayhVXXMG2bdtc3qOiooL77ruP+Ph4wsPDGTVqFAcOHGjQOptaom7fISIi0mDqFYgiIiLIz88HYOHChQwZMgSAkJAQysrKGqy4goICLr30UgIDA/n888/Zvn07f/vb34iOjjbbTJ8+nRkzZjB79mzWrFmDzWZj6NChFBUVmW0yMjKYP38+c+fOZdmyZRQXFzNixAgcDkeD1drUajdn1BoiERGRCxdQnxcNHTqUO+64g969e7Nz506uvfZaALZt20bbtm0brLjnnnuOlJQU3n77bfPYz9/fMAxeeOEFHn/8cW644QagZn1TUlISc+bM4a677sJut/PWW2/x3nvvmcHt/fffJyUlhcWLFzN8+PDTfnZFRQUVFT+NvhQWFjZYvxqCbt8hIiLScOo1QvTyyy/Tv39/jhw5wocffkhcXBwA69atY+zYsQ1W3EcffUSfPn349a9/TWJiIr179+aNN94wz2dmZpKbm8uwYcPMY8HBwQwaNIjly5ebNVVVVbm0SU5Oplu3bmab05k2bRpWq9V8pKSkNFi/GoLWEImIiDSceo0QRUdHM3v27FOOP/XUUxdc0M/t3buXV199lQcffJD/+Z//YfXq1UyePJng4GBuvfVWcnNzAUhKSnJ5XVJSknm1W25uLkFBQcTExJzSpvb1p/PYY4/x4IMPms8LCws9KhTVjhDZy6oor3IQEujv5opERES8V71GiL744guWLVtmPn/55Zfp1asX48aNo6CgoMGKczqdXHzxxUydOpXevXtz11138dvf/pZXX33VpZ3FYnF5bhjGKcdOdq42wcHBREVFuTw8SVRoAEEBNT8+LawWERG5MPUKRFOmTDHX1GzZsoWHHnqIa665hr1797qMqlyoFi1a0KVLF5djnTt3JisrCwCbzQZwykhPXl6eOWpks9morKw8Jaj9vI03slgsmjYTERFpIPUKRJmZmWZQ+fDDDxkxYgRTp07llVde4fPPP2+w4i699FJ+/PFHl2M7d+6kTZs2AKSmpmKz2Vi0aJF5vrKykiVLljBgwAAA0tPTCQwMdGmTk5PD1q1bzTbeqoW1ZtrsQEGpmysRERHxbvUKREFBQZSW1nwJL1682FywHBsb26BXYz3wwAOsXLmSqVOnsnv3bubMmcPrr7/OpEmTgJpRkoyMDKZOncr8+fPZunUrt99+O2FhYYwbNw4Aq9XKxIkTeeihh/jqq6/YsGEDt9xyC927dzevOvNWHW2RAGzP8awr4ERERLxNvRZVDxw4kAcffJBLL72U1atX88EHHwA1ozetWrVqsOL69u3L/Pnzeeyxx3j66adJTU3lhRdeYPz48WabRx55hLKyMu655x4KCgro168fCxcuJDIy0mwzc+ZMAgICGDNmDGVlZQwePJh33nkHf3/vXojcuUXNuqYdOUXnaCkiIiJnYzEMwzjfF2VlZXHPPfeQnZ3N5MmTmThxIlAzouNwOHjppZcavFB3KywsxGq1YrfbPWaB9YasAq5/ZTkJkcGsedy7R7tEREQaQ12/v+s1QtS6dWs++eSTU47PnDmzPm8n9dTRFonFUnOV2ZGiChJOLLIWERGR81OvQATgcDhYsGABO3bswGKx0LlzZ0aPHu3101DeJCwogNS4cPYeLWFHTiEJkQnuLklERMQr1SsQ7d69m2uuuYaDBw/SsWNHDMNg586dpKSk8Omnn9K+ffuGrlPOoHOLKDMQXX6RApGIiEh91Osqs8mTJ9O+fXuys7NZv349GzZsICsri9TUVCZPntzQNcpZdEmuXVitK81ERETqq14jREuWLGHlypXExsaax+Li4nj22We59NJLG6w4ObfOLWquptOVZiIiIvVXrxGi4OBgiopO/QIuLi4mKCjogouSuqu99H7PkWLKqxxurkZERMQ71SsQjRgxgjvvvJNVq1ZhGAaGYbBy5UruvvtuRo0a1dA1ylnYokKIDguk2mmwO6/Y3eWIiIh4pXoFopdeeon27dvTv39/QkJCCAkJYcCAAXTo0IEXXnihgUuUs7FYLHQ5MUqkHatFRETqp15riKKjo/nvf//L7t272bFjB4Zh0KVLFzp06NDQ9UkddG4RxfI9+VpYLSIiUk91DkTnuov9t99+a/55xowZ9S5Izt9Pt/BQIBIREamPOgeiDRs21KmdxWKpdzFSP7VXmm0/VIhhGPoZiIiInKc6B6JvvvmmMeuQC5CWGEmgv4XC8moO2ctpGR3q7pJERES8Sr0WVYtnCQrwo31CBAA7D2s/IhERkfOlQOQjWsXUjAodOl7m5kpERES8jwKRj2hhrQlEufZyN1ciIiLifRSIfITNGgLAoeMKRCIiIudLgchHJEfXBKIcu6bMREREzpcCkY/QlJmIiEj9KRD5iBa1U2b2MgzDcHM1IiIi3kWByEckRdUEovIqJ8dLq9xcjYiIiHdRIPIRIYH+xIUHAZCjaTMREZHzokDkQ1poYbWIiEi9KBD5kNqF1Yc0QiQiInJeFIh8SO3C6lyNEImIiJwXBSIfUjtClKPNGUVERM6LApEPqd2c8ZBGiERERM6LApEPsUXVTplphEhEROR8KBD5kOToE1Nm9nJtzigiInIeFIh8SO3mjBXVTo6VVLq5GhEREe+hQORDggL8iI8IBrQ5o4iIyPlQIPIxP931XoFIRESkrhSIfEztwmrtVi0iIlJ3CkQ+5ucLq0VERKRuFIh8TO1u1TnHNUIkIiJSVwpEPsZmrd2cUSNEIiIidaVA5GNqp8y0OaOIiEjdKRD5mJ9u8FqO06nNGUVEROpCgcjHJEWFYLFApcNJvjZnFBERqRMFIh8T6O9Hgrk5oxZWi4iI1IUCkQ9qlxAOwLr9BW6uRERExDsoEPmgIZ2TAPh8a66bKxEREfEOCkQ+6KpuNgDW7DvGkaIKN1cjIiLi+RSIfFCrmDB6tLJiGLBo+2F3lyMiIuLxFIh81PCuNaNEn2/NcXMlIiIink+ByEddfWLabMWefOylVW6uRkRExLMpEPmodgkRdEyKpNppsHiHps1ERETORoHIhw3vVjttpqvNREREzkaByIfVTpt9t+sIxRXVbq5GRETEcykQ+bBOtkjaxIVRWe1k2a6j7i5HRETEYykQ+TCLxcJlafEArNyb7+ZqREREPJcCkY8b0L4mEC3foxEiERGRM/GqQDRt2jQsFgsZGRnmMcMwePLJJ0lOTiY0NJQrrriCbdu2ubyuoqKC++67j/j4eMLDwxk1ahQHDhxo4urd45J2cQDsPFysXatFRETOwGsC0Zo1a3j99dfp0aOHy/Hp06czY8YMZs+ezZo1a7DZbAwdOpSioiKzTUZGBvPnz2fu3LksW7aM4uJiRowYgcPhaOpuNLnY8CC6tIgCYIWmzURERE7LKwJRcXEx48eP54033iAmJsY8bhgGL7zwAo8//jg33HAD3bp1491336W0tJQ5c+YAYLfbeeutt/jb3/7GkCFD6N27N++//z5btmxh8eLFZ/zMiooKCgsLXR7eakD7mlGiFZo2ExEROS2vCESTJk3i2muvZciQIS7HMzMzyc3NZdiwYeax4OBgBg0axPLlywFYt24dVVVVLm2Sk5Pp1q2b2eZ0pk2bhtVqNR8pKSkN3KumM6BDTSBavkcjRCIiIqfj8YFo7ty5rF+/nmnTpp1yLje3ZsPBpKQkl+NJSUnmudzcXIKCglxGlk5uczqPPfYYdrvdfGRnZ19oV9ymb9tY/P0s7M8v5UBBqbvLERER8TgeHYiys7O5//77ef/99wkJCTljO4vF4vLcMIxTjp3sXG2Cg4OJiopyeXiryJBAerSyAjX3NhMRERFXHh2I1q1bR15eHunp6QQEBBAQEMCSJUt46aWXCAgIMEeGTh7pycvLM8/ZbDYqKyspKCg4Y5vm4Kd1RDWBaHdekfYmEhEROcGjA9HgwYPZsmULGzduNB99+vRh/PjxbNy4kXbt2mGz2Vi0aJH5msrKSpYsWcKAAQMASE9PJzAw0KVNTk4OW7duNds0B7X7ES3ZeYQJb61iyIzvuPn1lfzpv1updjjdXJ2IiIh7Bbi7gLOJjIykW7duLsfCw8OJi4szj2dkZDB16lTS0tJIS0tj6tSphIWFMW7cOACsVisTJ07koYceIi4ujtjYWB5++GG6d+9+yiJtX5beJoYgfz/ySypZuusofhZwGvDuiv1kHStl1riLiQj26P8cREREGo3XfwM+8sgjlJWVcc8991BQUEC/fv1YuHAhkZGRZpuZM2cSEBDAmDFjKCsrY/Dgwbzzzjv4+/u7sfKmFRLoz9hfpPDpllyu65XMbQPasu2QnYwPNvLNj0cY89oK5t0zgJDA5vN3IiIiUstiGIbh7iK8QWFhIVarFbvd7tULrE+2Kfs4t729muOlVbxxax+Gdmk+66pERMT31fX726PXEEnj65kSzVVdbQCs3X/MzdWIiIi4hwKRkN6mZo+mdfsKztFSRETENykQCX3axgKw+aCdimrfv7+biIjIyRSIhLZxYcSFB1FZ7WTrQbt5/ONNh5i5aCdaZiYiIr5OgUiwWCzmtNnaE9Nm9tIqHvr3Jl78aherM7W2SEREfJsCkQDQp21NIFpzIhB9tOkgldU1Gzau3a+1RSIi4tsUiASA9DY164jWZxVgGAYfrP3pZrZr92mESEREfJsCkQDQrWUUQQF+HCup5NMtOWw9WGieW7e/AKdT64hERMR3KRAJAMEB/vRqFQ3Anz/ZDsCwLkmEBflTWF7NrrxiN1YnIiLSuBSIxJR+Yh3R4cIKAMb2a02vlGhAmzaKiIhvUyASU58TV5oB2KJCuDwtwdyjSJs2ioiIL1MgElP6zwLRjekt8fezmCFpjUaIRETEhykQiSk6LIhLO8QRGRLAzX1bA9C7dTR+Fsg+VkZeYbmbKxQREWkcAe4uQDzLW7f1paLKiTUsEIDIkEA62qLYkVPI2v0FXNO9hZsrFBERaXgaIRIXIYH+Zhiq1eekXawNw6DK4Wzy2kRERBqLApGcU+0u1sv3HOXlb3ZzybSv6D/ta3YdLnJzZSIiIg1DgUjOqXax9Q+5RTz/5Y8cLqzgaHEFt7+9hrwirSsSERHvp0Ak59QyOpROtkgAuiZH8dyN3UmND+fg8TLueHctpZXVbq5QRETkwlgMw9A9GeqgsLAQq9WK3W4nKirK3eU0uaPFNaNCHZMisVgs7DtawvWvfE9BaRVDuyTx+oR0LBaLu8sUERFxUdfvb40QSZ3ERwTTyRZlhp628eG8cWsfggL8WLT9MCv25Lu5QhERkfpTIJJ669M2ll+ltwLgg7XZbq5GRESk/hSI5ILc1CcFgM+35mIvrXJzNSIiIvWjQCQXpEcrK51skVRWO/nvpoPuLkdERKReFIjkglgsFsacGCX6YI2mzURExDspEMkFu753S4L8/dh2qJCtB+2nnC+vclBSoUvzRUTEcykQyQWLCQ9iWNckAP7vpMXV+/NLGPy3JQx6/hutMRIREY+lQCQN4qa+NdNm8zccNC/Bzz5Wyrg3VnHweBlHiytZsFFrjERExDMpEEmDuLR9PJ1skRSVVzP2jZXcP3cDY99YycHjZYQE1vxn9q/VWWgfUBER8UQKRNIg/PwsfHBnf265pDUWC/x34yEOFJTRNi6Mj+8dSHCAHz/kFrEh+7i7SxURETmFApE0GGtYIH+5rjsL7rmUvm1j6NYyijm/vYS0pEiu7dECgLmrs9xcpYiIyKkUiKTB9UyJ5t93D+CT+y4jOToUgHG/aA3Ax5tyKCzX4moREfEsCkTSJNLbxJCWGEFZlYP/bjzk7nJERERcKBBJk7BYLNx8YpTo7e8z+WrHYYo0UiQiIh5CgUiazA29WxIS6MfeIyVMfHctvZ5exB3vrsVepmAkIiLupUAkTSYmPIh//KYfY3/RmjZxYTicBot3HOamv68gr7Dc3eWJiEgzZjG0MUydFBYWYrVasdvtREVFubscn7DlgJ3fvLuGI0UVtI4N4/2J/WgdF+buskRExIfU9ftbI0TiNt1bWfnP3f1pHRtG1rFSbn59BWWVDneXJSIizZACkbhVm7hw/nN3f5KtIRyylzNvwwF3lyQiIs2QApG4XWJUCHdc1g6At5Zl4nRqFldERJqWApF4hDF9U4gMCWDvkRK+3ZkHgMNp8LeFP/LyN7vdXJ2IiPg6BSLxCBHBAeZu1m98l4lhGDz18TZmfb2b57/8kY26B5qIiDQiBSLxGLcNaIu/n4UVe/OZ8p/N/GPFfvPc/y7LdGNlIiLi6xSIxGMkR4dybfeam8D+Z13N4urx/WpGjT7bkkOOvcxttYmIiG9TIBKPcsdlqeaf7xrUjmeu706/1FiqnQbvLt9/lleKiIjUnwKReJQeraL5w4guTBnekUeHdwJg4sCakPSv1VmUVla7szwREfFRAe4uQORktQGo1uDOSbSJC2N/fikfrj/IhEvauKkyERHxVRohEo/n72fh/w1oC8BbS/dSXqXdrEVEpGEpEIlX+FWfFOLCg9iXX8rTn2x3dzkiIuJjFIjEK0QEBzDzpl5YLDBnVRYLNhx0d0kiIuJDFIjEa1x+UQL3/TINgMfmbWHX4aJT2lRUOzAM3fpDRETOj0cHomnTptG3b18iIyNJTEzkuuuu48cff3RpYxgGTz75JMnJyYSGhnLFFVewbds2lzYVFRXcd999xMfHEx4ezqhRozhwQDcR9Ub3D05jYId4yqocDHvhO3o8+SUDn/uaAdO+ovMfvqDjE19w3SvLqax2urtUERHxIh4diJYsWcKkSZNYuXIlixYtorq6mmHDhlFSUmK2mT59OjNmzGD27NmsWbMGm83G0KFDKSr6afQgIyOD+fPnM3fuXJYtW0ZxcTEjRozA4dDiXG/j72fhhZt70TEpEsOAwvJqDhSUccheTtmJxdabso/zlna2FhGR82AxvGh+4ciRIyQmJrJkyRIuv/xyDMMgOTmZjIwMHn30UaBmNCgpKYnnnnuOu+66C7vdTkJCAu+99x433XQTAIcOHSIlJYXPPvuM4cOH1+mzCwsLsVqt2O12oqKiGq2PUjcOp0F+SQWFZdXYy6rw97MQExbI97vz+Z/5WwgN9GfxQ4NoGR3q7lJFRMSN6vr97dEjRCez2+0AxMbGApCZmUlubi7Dhg0z2wQHBzNo0CCWL18OwLp166iqqnJpk5ycTLdu3cw2p1NRUUFhYaHLQzyHv5+FxMgQOiRGkN4mhl4p0bSJC2fsL1L4RdtYyqocPP3xtnO/kYiICF4UiAzD4MEHH2TgwIF069YNgNzcXACSkpJc2iYlJZnncnNzCQoKIiYm5oxtTmfatGlYrVbzkZKS0pDdkUZisVh4+rqu+PtZ+HLbYb75Ic/dJYmIiBfwmkB07733snnzZv71r3+dcs5isbg8NwzjlGMnO1ebxx57DLvdbj6ys7PrV7g0uU62KHMjxyc/3qYF1iIick5eEYjuu+8+PvroI7755htatWplHrfZbACnjPTk5eWZo0Y2m43KykoKCgrO2OZ0goODiYqKcnmI98gYehEJkcHszy/lX6uz3F2OiIh4OI8ORIZhcO+99zJv3jy+/vprUlNd73GVmpqKzWZj0aJF5rHKykqWLFnCgAEDAEhPTycwMNClTU5ODlu3bjXbiO+JCA5g8uCaPYte+moXxRW6KayIiJyZR9/cddKkScyZM4f//ve/REZGmiNBVquV0NBQLBYLGRkZTJ06lbS0NNLS0pg6dSphYWGMGzfObDtx4kQeeugh4uLiiI2N5eGHH6Z79+4MGTLEnd2TRnZz3xTeWrqXffmlvLl0LxlDLgJqgvbWg4X839psPt58iK7JUfzv7X0JDvB3c8UiIuIuHn3Z/ZnW+Lz99tvcfvvtQM2X21NPPcXf//53CgoK6NevHy+//LK58BqgvLycKVOmMGfOHMrKyhg8eDCvvPLKeS2U1mX33unTzTlMmrOe8CB/Fj04iO93H+Wd5fvYdsj1qsHx/VrzzPXd3VSliIg0lrp+f3t0IPIkCkTeyTAMrnv5ezYdsBPgZ6HaWfOfe5C/H8O72ejR0srUz3dgGPD8r3rw6z71u5qw2uEkwN+jZ6BFRJqlun5/e/SUmciFslgsPHpVJ8a9uYpqp0ELawi39m/LzX1TiAkPAqC00sHMxTt5YsFWOreIoltL63l9xv78Eka//D2/7JjIjJt6NUIvRESksWmEqI40QuTdFm7LxWkYDO6cROBJIzlOp8HEd9fwzY9H6JAYwaIHLj/ntg0/9/TH2/nf7zMJDvBjy5PDCQrQSJGIiKfwyZ2qReprWFcbV3VrcUoYAvDzszDzpl6EBfmzO6+YdfsLTvMOp1de5eA/62r2qKqodvJDrnY0FxHxRgpEIkB0WBBXd2sBwH/WHajz6z7dnENh+U+X9K8/jzAlIiKeQ4FI5IQb01sCNSGnvMpx2jbFFdX8fJb5n6v2AxAfEQzAhuzjjVukiIg0Ci2qFjnhktQ4WkaHcvB4GV9uy2V0r5Y4nAb/WLGPZbuOsu1QIbmF5XRuEcUr4y+mvMrB+qzjBPhZ+J9rOvHg/21iQ9Zxd3dDRETqQYFI5AQ/Pws3XtySl77ezYfrDzKqZzJ//mQ77yzf59JuR04ho2Yvo0uLmsV5w7vaGNIlCYsFso6VcrS4whwxEhER76ApM5GfueHimnvlLdt1hGc+3WGGoYwhafz77v588/AVpLeJoai8mlWZx4CaTR2jQgLpkBABoFEiEREvpEAk8jNt48Pp0yYGpwFvLssE4IlrO5Mx5CL6to0lNT6cf/32Em65pDUAnWyR9G8fB8DFrWMA2JD108LqvKJyKqpPXY+072gJJbq/moiIx1AgEjnJjemtzD9PHJjKHZe1czkfFODHX67rzuf3X8ac315i7lnUu3U0AOtPBKKVe/O59NmvmfDmaqodTvP1X2zN4cq/fcv9czc2bkdERKTOFIhETjKqZzKXpcVz+4C2PH5N5zO269wiitgTu10D9D4xQrT5gJ2Simp+/+FmqhwGq/cd4/WlewHIL67g8flbMQz4+ofDHCupbNzOiIhInSgQiZwkPDiA9yb248lRXfHzq/uO1R0SI4gIDqC00sH9czewL7+UkMCaf2IvLNrFj7lF/OmjbeSfCEFOAxbvOGy+vqLawUtf7WKjLt0XEWlyCkQiDcTfz0KvlGgAFu/IA+DFm3szuFMilQ4nE95axSebc/D3s3BVVxtQc0uRWu98v48Zi3by23+spVjri0REmpQCkUgDql1HBHB1NxvDu9qYdkN3rKGB5BVVAPC7Qe3JGJoGwHe7jlJSUU21w8m7J65oO1JUwWvf7mnq0kVEmjUFIpEG1KdtLACRIQE8NaorAIlRITw9uubPnWyR3De4Ax2TImkTF0ZltZMlO4/wxbZcDtnLzRvDvr50LwcKSgFYvP0wlz77NU9/vB2nU/diFhFpDNqYUaQBXZ4Wz5Mju9AzJZrEqBDz+OheLWmfEEFKTBjBAf5AzYaOr3+3ly+35ZJ1rCb83D2oPasz81m59xjPffEj/VJj+eN/t+I04H+/z+R4aSXTf9WDgNPcpFZEROrPYvz8xkxyRoWFhVitVux2O1FRUe4uR3zAuv3HuPHVFQT6W6hyGAT5+/H9739JXlE5I2Yt4+f/Mi9Li2f5nnwcToNru7fghZt7EXiBoWjf0RJaxYQqXImIT6vr97d+E4q4Se+UGBIig6ly1CSfUb2SSYgMpmuylTHpKWa7B4ZcxD9+8wteGX8xgf4WPt2SQ8YHG12mz7KPlTLpn+v5/YebeWtZJt/vPuqy99HJ/vrlj1zx128Z/fL37M4rbrxOioh4CU2ZibiJn5+FoV2SmLMqC4DfXJpqnnvsmk4EBljo3y6ea3u0AE5Msd3ahzv/sZZPN+eQGhfOw8M7kmMvY+wbKzlQUOby/pe0i+Wt2/oSHuz6z/w/6w4w+5vdAGw7VMiIWUv508iu3Nw3xdxkUkSkudGUWR1pykwaw9p9x/j131fwy46JvHV73zq95j/rDvDwvzcBNbcVmbM6i71HSmgTF8aonsnsPFzEsl1HKal00LdtDG//v18QcSIUrc48xvg3V1LlMLh9QFt25xWzbPdRAOIjgvlFagz9UuO4/uKWRIUENk6nRUSaUF2/vxWI6kiBSBrL/vwSEiNDCA3yr/Nrpn/xA6/87NL8ZGsI/3d3f1rFhAGwMfs4t761isLyanq3jmZUz2T255fy340HKSit4pruNmaPvRiAN5ftZeaiXZRV/XTPtZTYUF4dn063ltYG6qWIiHsoEDUwBSLxJE6nwb3/Ws9nW3KJjwjm/+66hHYJES5tth60M/7NVdjLqlyO92xlZe6d/V0CWEW1g80H7KzOPMbcNVlkHysjKMCPP47oQucWkew7WsrB42VUVjtxGAb+FgtXdbO5BKYqh5P9+SW0T4jQ1JuIeAwFogamQCSeprzKwcebDtG/fZw5MnSyHTmFvLB4JxYstIkLIzU+nJE9k09ZV/Rz9tIqHvr3RnO37bO5/KIEbu6bwpp9x/ho4yHySyoZ3SuZv/26p8vVa1UO5wVfFSciUh8KRA1MgUiaE6fT4O/f7eX17/YQFhRAm7gwWsWEEhYUgJ/FQm5hGV9szeVM+0Re1dXGS2N7c7iwnGc+3cGiHYf51cWteOyaTkSH/XRD3GqHU5f9i0ijUiBqYApEIq6y8kv5+3d7WLLzCL1Sornh4pZUVhtM/tcGKh1OurSIYveRYiqrf7r8Pz4iiPt+mcah42V8/UMeu/KKaZ8QTnqbGLomW6lyOCmuqMZpwC87JdKzlRWLxcLR4greXb6PTQfsPHFtZy5Kimyyfn614zDL9+QzeXAa1lAtNBfxNgpEDUyBSKRuluw8wp3/WEvFiSB0aYc4burbmpe+2nXeex51SIygW3IUn2/NNd+vZXQo8ycNIDEy5JT2mw8cZ/GOPG4f0JbY8KBTzp+vZbuOctvbq3E4Dfq2jeEfv+l3XovfRcT9FIgamAKRSN2t2XeMt7/PZHSvlgzrkoTFYqGi2sHrS/byxbZcOtoiubJjIr1Sotl5uIh1+wvYlVdMaKA/ESEBFJZVsWj7YTMEQc1icHtZFfvyS+mVEs3cOy8hJPCncLJybz7/7+01lFU5aJcQzvsT+5EcHVrvPuw5Usz1L39PYXm1eWxwp0Rem5Cu9VAiXkSBqIEpEIk0raLyKj7bksMPuUUM72qjX2os+/JLuf6V7zleWsW13Vsw9YbuWEMDXcKQv58Fh9Mg2RrCPyb2o0PiT1ffOZwG+/JLKKt0nHVLgbyicsa8toJ9+aWkt4nhgSEXMfHdNVRUO7nh4pb89Vc98fPTlXQi3kCBqIEpEIl4hpV785nw1irzlicdEiM4WFBGWZWDyy9K4KlRXZn47hr2HikhIjiAVjGhhAT6U+Vwsjuv2Bx1un9wGg8MvQgAwzD4x4r9zF2TzcGCUnNUqGV0KP+991LiI4JZvP0wd72/DofT4PreLZn+qx7mSNE3P+ax/2gJ4y9p4zJ6lH2slK92HOZocSX5JRXEhgdx5+XtG2UtkmEYGAYKaiInUSBqYApEIp7ji605PPv5D+zLLzWPXX5RAq9PSCck0J/84gr+3ztr2HzAfsprQwL9KK+qCUUZQ9K454oOPD5/C/9ed8ClXbv4cF655WI62X769/7fjQd58P824XAa/LJTIk9c25lnP/+BhdsPA3Bdr2RmjOmFn5+FjdnHmfDmKooqql3et2V0KDPG9KRfu7gG+/vYd7SESXPWU17l4B8T+9HyAqYKRXyNAlEDUyAS8Tz5xRVsyDrOkeIKru/d0mVNUbXDyaYDdkoqqimvcmCxWEhLjKB1bBhvLcvkmc92ANA6NoysY6X4WWDK8E4M6ZxIcnToGfdq+vqHw9zzz/VmqAIIODEqU+00GNevNb9Ob8Wtb62mqKKazi2i6Ns2htjwIOZvOMj+/FIsFri6m43Csmr2HyuhtMJBdFggMWFBpCVFcOfl7UmNDzffP9deTkFpJR0SI05Zv7RiTz53v7/O3ICzXUI4/7l7QIMsKhfxBQpEDUyBSMS3vP7dHqZ+9gMAEcEBzBrXmys7JtbptWv3HeM376yhsLya9DYxTL2+OzsPFzF57gYMoyYgVTsNfpEay9u3/3SD3eKKap7+eBv/t/bAWd/f38/CmD4p9G4dzYINB1mxNx/DgOAAP7q1tNIuPpzgQD8cTvj32myqnQY9W1k5UlTBIXs5PVOimXNHPwB25xUTHRZIm7jws37mhSirdPD51hw+2nSItnHh/P7qTi7hVMSdFIgamAKRiO/556r9fL0jj0ev7nTeexsdKChld14xl6clmOt2PliTxaMfbgE4JQz93JKdR9iQVUDL6FBax4YRGRLI8bJKjpVU8uG6A3zz45FTXhMRHEDxSdNvtUb2TOb5X/XgQEEpv3ptBcdLq4gKCXC5Qu7KjgncNag9FyVFsmJPPqsy84kLD+Y3A9sSeZob+RqGQebREnYeLmLPkRL2HimhZXQIE/q3JSEyGKiZqntj6V7+u/GQS20Xt47mtQnpp90a4Xw4nAYr9+ZTUe3AFhVKC2sIMRr5kvOkQNTAFIhEpC4+2XyILQftTP5l2llvkXI2a/Yd46WvdnGkqIKru7Xghotb0jI6lH35JWw6cJxDx8uprHZSUe0kLTGCGy5uad4/bkNWAePeWGXerDcuPIhjpZWc6Td9fEQwj13diSGdk9h04Dgbso6zIbuAjdnHOV5adUr74AA/xvRJwV5WxSebD5m7lbeODeOqbjbmrs6isLyaFtYQ/ueaziRGBhMdFkRqfDhBAWferqD2q8hisVDtcPLJ5hxmfb2LPUdKXNpdflECL97US8FI6kyBqIEpEImIt8jKLyXHXkZaUiSx4UHsO1rCm8v28u+1B6iodtIxKZJL2sWydNdR9h4tOeP7BAX40ckWSYeECNrEhfPtzjw2ZB13aXNlxwR+e3k7LkmNw8/Pwt4jxdzxj7XsPSnItLCG8Mz13fhlpySX49nHSnlv5X7+b2029rIqQgP9sQAllTWBLiokgJTYMHLt5eSXVALQJi6Mt27rQ4fEM4/q5RWV8591B9iZW0TWsVLyiioY3CmRx67pbE7nlVRU8+W2mhsk92sXS3DAhU/zLdl5hIf/vYkeLa3cckkbLr8oAf8TI4iGYZxy4+OSimq++TGPy9IStBN6I1EgamAKRCLi7YorqqmsdpoLriurnfzv95m89NUuSisdtIkLo3dKNL1bx9C7dTSdbFEuozqGYbBibz7vr9xPSKA/Ewem0jX51P2c7GVVPPv5D2zPKaSwrIojRRXmlNrInsn8slMCOw8Xs/WgnWW7j5529ComLJA7LmvHrf3bmFN6P+QWcse7azlQUEZkcAAD0+LJLijlYEEZrWPDGHRRAn3axvLltlz+ve6Ay21janVMimT2uN5szylk6mc7OFxYAUBYkD+XpcUzsmcyw7rYzjqa9fO/j58HnFx7OVe/+B0FPxtZS7aGEBrkT35JJRVVTv7fpW15eFhH/PwsFJVXcev/rmZD1nE62SKZe+clLvf6Ox9VDie7DhfTyRZ5ytYLpwtizYkCUQNTIBIRX1VW6aC8ytFo01BllQ5mLt7Jm0v3nvaGwJelxXP7gLZ0b2WlospJeZWDlNiw0y7Mzi+u4O7317FmX8E5P7d362iGdbHRJi4Mh9PgqY+3c7S4AosFM4S1jA6lyuEkr6jCfF1ceBDX926J06gJYVnHShnYIZ77h6TRwhpKYXkVs7/ezZxVWQzrmsSTo7oSHhTAuDdWsirzGN1aRtEvNY5/r812WcdV69ruLXh6dFfufG8d6/b/1I8eray8f0c/ok4EQKfTwGLhnGFmzb5j/M+8LezKK+aSdrHMGNOL5OhQjpVUMvWzHXy+JYdRvZK554oOpMSGnfJ6h9OgstrZJLeleef7TL7bdZRHr+pER1vT3JNQgaiBKRCJiFyYLQfsPL/wR0orquloi6SjLZJLO8TTPiHi3C/+mcpqJx+uP0BZpYPWsWHYrCH8kFvEkp1HWLfvGGlJkfzuivb0S411CRN5ReU8+MEmlu0+SmigP5OubM8dl7UjyN+PbYcK+WJbDv9ee8AlHP1ccIAf1/VqyeIdh83pO6gJVf3bx/GfdQcID/Ln08mX0TY+nLJKB6sy8wkO8Cc+IogN2cd5fP4WqhyGuR9WVEgAf76uG099vJ1jJZX0bh1N1+Qo1u0/zo+5hTgN8LNAaKA/gzsnMaF/G/q0iaHKYbAjp5C5a7L41+pslzqjQgK45ZI2/Gt1lstoVYCfhZE9k4kOC6S8yklhWRV7jhSz92gJ1Q4ntw1oy6NXnfsKwRV78lm+5yi3/myBfV28/M1unv/yR6CmP9N/1YORPZMpqajm8625fLr5EK+MT2/wYKZA1MAUiEREvJ/TafDdriN0skVhs556FVy1w8lXP+Tx5dZcosOC6NQiktiwIP7+3R6XUal2CeH8vwFteX3pXrKPlZnHX7ipF9f1bnnGz1+xJ5+73ltLYXk1kSEB/POOfvRoFc3Wg3bGvbHytCNKJ2sZHcqR4gqXKcGb+6Zw8y9a86ePtrEp+7h5vJMtkjsvb8e89QdZtvvoOd+7fUI4T47qyrGSStbsO0auvYKhXRIZ0SMZiwWe/fwH/rFiPwDxEUH8bUwvBl2UcM73feO7vebeX+0Tws3F8gPax7Ex+zilJ9aMvXhzL0b3OvPfX30oEDUwBSIRkebLMAy+2pHHv1ZnMTAtnltO3KalqLyKJz/azofrDzD2F62ZdkP3c77X7rwi3l+Zxa/7tHJZg7X5wHFe+moXbeLC6dMmhh4p0YQE+OFwGhw8Xsbc1dn8d9NBc1PQ6LBALm4dw12XtzN3Pq9yOJn19W4WbDjIuH6tmTgw1dzMc+2+Yyzafhg/PwshAf6EB/uTGh9OWmIke44U8+iHm884OhYRHIA1NJCDx2vCny0qhNzCcgCu792SskoHO3ILOV5aRb/UWK7slEhHWyTbDhWyJvMYH206BMADQy7i3l924K8Lf+TVb/eY758aH86NF7fkxvRWtLA27E7rCkQNTIFIRETO5FhJJTFhgY2+eNleVsWGrALaxoXTJi6sQT/veGklf/poG4u2H6ZDYgR928YSHRrIh+sPmLfJaWENYfqvetC3bSxTP9thjhbVxX2/7MBDwzqaz7/+4TAr9x5jeFcbF7eObrS/OwWiBqZAJCIizcHJV6U5T2yQuSO3iF+lt3LZHuCbH/P4ekcerWPD6JIcRXhwAN/tPMLXP+RxoKCUzi2i6J0STf/28VzSLtYtV7spEDUwBSIRERHvU9fv73NvtCAiIiLi4xSIREREpNlTIBIREZFmT4FIREREmj0FIhEREWn2FIhERESk2VMgEhERkWZPgUhERESavWYViF555RVSU1MJCQkhPT2dpUuXurskERER8QDNJhB98MEHZGRk8Pjjj7NhwwYuu+wyrr76arKystxdmoiIiLhZs7l1R79+/bj44ot59dVXzWOdO3fmuuuuY9q0aed8vW7dISIi4n10646fqaysZN26dQwbNszl+LBhw1i+fPlpX1NRUUFhYaHLQ0RERHxTswhER48exeFwkJSU5HI8KSmJ3Nzc075m2rRpWK1W85GSktIUpYqIiIgbNItAVMtisbg8NwzjlGO1HnvsMex2u/nIzs5uihJFRETEDQLcXUBTiI+Px9/f/5TRoLy8vFNGjWoFBwcTHBxsPq9daqWpMxEREe9R+719riXTzSIQBQUFkZ6ezqJFi7j++uvN44sWLWL06NF1eo+ioiIATZ2JiIh4oaKiIqxW6xnPN4tABPDggw8yYcIE+vTpQ//+/Xn99dfJysri7rvvrtPrk5OTyc7OJjIy8ozTbN6qsLCQlJQUsrOzm8UVdOqv72pOfQX119c1p/42Zl8Nw6CoqIjk5OSztms2geimm24iPz+fp59+mpycHLp168Znn31GmzZt6vR6Pz8/WrVq1chVuldUVJTP/6P7OfXXdzWnvoL66+uaU38bq69nGxmq1WwCEcA999zDPffc4+4yRERExMM0q6vMRERERE5HgUgIDg7mT3/6k8tVdb5M/fVdzamvoP76uubUX0/oa7O5dYeIiIjImWiESERERJo9BSIRERFp9hSIREREpNlTIBIREZFmT4GomZg2bRp9+/YlMjKSxMRErrvuOn788UeXNoZh8OSTT5KcnExoaChXXHEF27Ztc1PFDWvatGlYLBYyMjLMY77W34MHD3LLLbcQFxdHWFgYvXr1Yt26deZ5X+lvdXU1TzzxBKmpqYSGhtKuXTuefvppnE6n2cab+/rdd98xcuRIkpOTsVgsLFiwwOV8XfpWUVHBfffdR3x8POHh4YwaNYoDBw40YS/q7mz9raqq4tFHH6V79+6Eh4eTnJzMrbfeyqFDh1zew1f6e7K77roLi8XCCy+84HLcW/pbl77u2LGDUaNGYbVaiYyM5JJLLiErK8s835R9VSBqJpYsWcKkSZNYuXIlixYtorq6mmHDhlFSUmK2mT59OjNmzGD27NmsWbMGm83G0KFDzfu4eas1a9bw+uuv06NHD5fjvtTfgoICLr30UgIDA/n888/Zvn07f/vb34iOjjbb+Ep/n3vuOV577TVmz57Njh07mD59Os8//zyzZs0y23hzX0tKSujZsyezZ88+7fm69C0jI4P58+czd+5cli1bRnFxMSNGjMDhcDRVN+rsbP0tLS1l/fr1/OEPf2D9+vXMmzePnTt3MmrUKJd2vtLfn1uwYAGrVq067e0mvKW/5+rrnj17GDhwIJ06deLbb79l06ZN/OEPfyAkJMRs06R9NaRZysvLMwBjyZIlhmEYhtPpNGw2m/Hss8+abcrLyw2r1Wq89tpr7irzghUVFRlpaWnGokWLjEGDBhn333+/YRi+199HH33UGDhw4BnP+1J/r732WuM3v/mNy7EbbrjBuOWWWwzD8K2+Asb8+fPN53Xp2/Hjx43AwEBj7ty5ZpuDBw8afn5+xhdffNFktdfHyf09ndWrVxuAsX//fsMwfLO/Bw4cMFq2bGls3brVaNOmjTFz5kzznLf293R9vemmm8x/t6fT1H3VCFEzZbfbAYiNjQUgMzOT3Nxchg0bZrYJDg5m0KBBLF++3C01NoRJkyZx7bXXMmTIEJfjvtbfjz76iD59+vDrX/+axMREevfuzRtvvGGe96X+Dhw4kK+++oqdO3cCsGnTJpYtW8Y111wD+FZfT1aXvq1bt46qqiqXNsnJyXTr1s3r+w81v7ssFos5+ulr/XU6nUyYMIEpU6bQtWvXU877Sn+dTieffvopF110EcOHDycxMZF+/fq5TKs1dV8ViJohwzB48MEHGThwIN26dQMgNzcXgKSkJJe2SUlJ5jlvM3fuXNavX8+0adNOOedr/d27dy+vvvoqaWlpfPnll9x9991MnjyZf/zjH4Bv9ffRRx9l7NixdOrUicDAQHr37k1GRgZjx44FfKuvJ6tL33JzcwkKCiImJuaMbbxVeXk5v//97xk3bpx5A1Bf6+9zzz1HQEAAkydPPu15X+lvXl4excXFPPvss1x11VUsXLiQ66+/nhtuuIElS5YATd/XZnVzV6lx7733snnzZpYtW3bKOYvF4vLcMIxTjnmD7Oxs7r//fhYuXOgyH30yX+mv0+mkT58+TJ06FYDevXuzbds2Xn31VW699VaznS/094MPPuD9999nzpw5dO3alY0bN5KRkUFycjK33Xab2c4X+nom9embt/e/qqqKm2++GafTySuvvHLO9t7Y33Xr1vHiiy+yfv36867d2/pbexHE6NGjeeCBBwDo1asXy5cv57XXXmPQoEFnfG1j9VUjRM3Mfffdx0cffcQ333xDq1atzOM2mw3glNSdl5d3yv+NeoN169aRl5dHeno6AQEBBAQEsGTJEl566SUCAgLMPvlKf1u0aEGXLl1cjnXu3Nm8WsOXfr5Tpkzh97//PTfffDPdu3dnwoQJPPDAA+ZIoC/19WR16ZvNZqOyspKCgoIztvE2VVVVjBkzhszMTBYtWmSODoFv9Xfp0qXk5eXRunVr8/fW/v37eeihh2jbti3gO/2Nj48nICDgnL+3mrKvCkTNhGEY3HvvvcybN4+vv/6a1NRUl/OpqanYbDYWLVpkHqusrGTJkiUMGDCgqcu9YIMHD2bLli1s3LjRfPTp04fx48ezceNG2rVr51P9vfTSS0/ZRmHnzp20adMG8K2fb2lpKX5+rr+6/P39zf/j9KW+nqwufUtPTycwMNClTU5ODlu3bvXK/teGoV27drF48WLi4uJczvtSfydMmMDmzZtdfm8lJyczZcoUvvzyS8B3+hsUFETfvn3P+nuryfva4Mu0xSP97ne/M6xWq/Htt98aOTk55qO0tNRs8+yzzxpWq9WYN2+esWXLFmPs2LFGixYtjMLCQjdW3nB+fpWZYfhWf1evXm0EBAQYzzzzjLFr1y7jn//8pxEWFma8//77Zhtf6e9tt91mtGzZ0vjkk0+MzMxMY968eUZ8fLzxyCOPmG28ua9FRUXGhg0bjA0bNhiAMWPGDGPDhg3mVVV16dvdd99ttGrVyli8eLGxfv1645e//KXRs2dPo7q62l3dOqOz9beqqsoYNWqU0apVK2Pjxo0uv7sqKirM9/CV/p7OyVeZGYb39PdcfZ03b54RGBhovP7668auXbuMWbNmGf7+/sbSpUvN92jKvioQNRPAaR9vv/222cbpdBp/+tOfDJvNZgQHBxuXX365sWXLFvcV3cBODkS+1t+PP/7Y6NatmxEcHGx06tTJeP31113O+0p/CwsLjfvvv99o3bq1ERISYrRr1854/PHHXb4gvbmv33zzzWn/rd52222GYdStb2VlZca9995rxMbGGqGhocaIESOMrKwsN/Tm3M7W38zMzDP+7vrmm2/M9/CV/p7O6QKRt/S3Ln196623jA4dOhghISFGz549jQULFri8R1P21WIYhtHw404iIiIi3kNriERERKTZUyASERGRZk+BSERERJo9BSIRERFp9hSIREREpNlTIBIREZFmT4FIREREmj0FIhEREWn2FIhExGO1bduWF154oc7tv/32WywWC8ePH2+0mkTEN2mnahFpMFdccQW9evU6rxBzNkeOHCE8PJywsLA6ta+srOTYsWMkJSVhsVgapIbz9e2333LllVdSUFBAdHS0W2oQkfMX4O4CRKR5MQwDh8NBQMC5f/0kJCSc13sHBQVhs9nqW5qINGOaMhORBnH77bezZMkSXnzxRSwWCxaLhX379pnTWF9++SV9+vQhODiYpUuXsmfPHkaPHk1SUhIRERH07duXxYsXu7znyVNmFouFN998k+uvv56wsDDS0tL46KOPzPMnT5m98847REdH8+WXX9K5c2ciIiK46qqryMnJMV9TXV3N5MmTiY6OJi4ujkcffZTbbruN66677ox93b9/PyNHjiQmJobw8HC6du3KZ599xr59+7jyyisBiImJwWKxcPvttwM1QXD69Om0a9eO0NBQevbsyX/+859Tav/000/p2bMnISEh9OvXjy1btpzzc0XkwikQiUiDePHFF+nfvz+//e1vycnJIScnh5SUFPP8I488wrRp09ixYwc9evSguLiYa665hsWLF7NhwwaGDx/OyJEjycrKOuvnPPXUU4wZM4bNmzdzzTXXMH78eI4dO3bG9qWlpfz1r3/lvffe47vvviMrK4uHH37YPP/cc8/xz3/+k7fffpvvv/+ewsJCFixYcNYaJk2aREVFBd999x1btmzhueeeIyIigpSUFD788EMAfvzxR3JycnjxxRcBeOKJJ3j77bd59dVX2bZtGw888AC33HILS5YscXnvKVOm8Ne//pU1a9aQmJjIqFGjqKqqOuvnikgDMEREGsigQYOM+++/3+XYN998YwDGggULzvn6Ll26GLNmzTKft2nTxpg5c6b5HDCeeOIJ83lxcbFhsViMzz//3OWzCgoKDMMwjLffftsAjN27d5uvefnll42kpCTzeVJSkvH888+bz6urq43WrVsbo0ePPmOd3bt3N5588snTnju5hto6Q0JCjOXLl7u0nThxojF27FiX182dO9c8n5+fb4SGhhoffPDBOT9XRC6M1hCJSJPo06ePy/OSkhKeeuopPvnkEw4dOkR1dTVlZWXnHCHq0aOH+efw8HAiIyPJy8s7Y/uwsDDat29vPm/RooXZ3m63c/jwYX7xi1+Y5/39/UlPT8fpdJ7xPSdPnszvfvc7Fi5cyJAhQ7jxxhtd6jrZ9u3bKS8vZ+jQoS7HKysr6d27t8ux/v37m3+OjY2lY8eO7Nixo16fKyJ1pykzEWkS4eHhLs+nTJnChx9+yDPPPMPSpUvZuHEj3bt3p7Ky8qzvExgY6PLcYrGcNbycrr1x0sW1J1+RdvL5k91xxx3s3buXCRMmsGXLFvr06cOsWbPO2L62vk8//ZSNGzeaj+3bt7usIzqT2vrO93NFpO4UiESkwQQFBeFwOOrUdunSpdx+++1cf/31dO/eHZvNxr59+xq3wJNYrVaSkpJYvXq1eczhcLBhw4ZzvjYlJYW7776befPm8dBDD/HGG28ANX8Hte9Tq0uXLgQHB5OVlUWHDh1cHj9fZwWwcuVK888FBQXs3LmTTp06nfNzReTCaMpMRBpM27ZtWbVqFfv27SMiIoLY2Ngztu3QoQPz5s1j5MiRWCwW/vCHP5x1pKex3HfffUybNo0OHTrQqVMnZs2aRUFBwVn3McrIyODqq6/moosuoqCggK+//prOnTsD0KZNGywWC5988gnXXHMNoaGhREZG8vDDD/PAAw/gdDoZOHAghYWFLF++nIiICG677TbzvZ9++mni4uJISkri8ccfJz4+3rzi7WyfKyIXRiNEItJgHn74Yfz9/enSpQsJCQlnXQ80c+ZMYmJiGDBgACNHjmT48OFcfPHFTVhtjUcffZSxY8dy66230r9/fyIiIhg+fDghISFnfI3D4WDSpEl07tyZq666io4dO/LKK68A0LJlS5566il+//vfk5SUxL333gvAn//8Z/74xz8ybdo0OnfuzPDhw/n4449JTU11ee9nn32W+++/n/T0dHJycvjoo49cRp3O9LkicmG0U7WIyM84nU46d+7MmDFj+POf/9xkn6sdrkXcS1NmItKs7d+/n4ULFzJo0CAqKiqYPXs2mZmZjBs3zt2liUgT0pSZiDRrfn5+vPPOO/Tt25dLL72ULVu2sHjxYq3NEWlmNGUmIiIizZ5GiERERKTZUyASERGRZk+BSERERJo9BSIRERFp9hSIREREpNlTIBIREZFmT4FIREREmj0FIhEREWn2/j/dPBd3Q6qSEgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.optim import SGD, Adam\n",
    "from copy import deepcopy\n",
    "\n",
    "# 将数据适配为PyTorch数据集\n",
    "class SeqLabelDataset(Dataset):\n",
    "    def __init__(self, tokens, labels):\n",
    "        self.tokens = deepcopy(tokens)\n",
    "        self.labels = deepcopy(labels)\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        return self.tokens[idx], self.labels[idx]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "    \n",
    "    # 将每个批次转化为PyTorch张量\n",
    "    @classmethod\n",
    "    def collate_batch(cls, inputs):\n",
    "        input_ids, labels, masks = [], [], []\n",
    "        max_len = -1\n",
    "        for ids, tags in inputs:\n",
    "            input_ids.append(ids)\n",
    "            labels.append(tags)\n",
    "            masks.append([1] * len(tags))\n",
    "            max_len = max(max_len, len(ids))\n",
    "        for ids, tags, msks in zip(input_ids, labels, masks):\n",
    "            pad_len = max_len - len(ids)\n",
    "            ids.extend([0] * pad_len)\n",
    "            tags.extend([0] * pad_len)\n",
    "            msks.extend([0] * pad_len)\n",
    "        input_ids = torch.tensor(np.array(input_ids),\\\n",
    "            dtype=torch.long)\n",
    "        labels = torch.tensor(np.array(labels), dtype=torch.long)\n",
    "        masks = torch.tensor(np.array(masks), dtype=torch.uint8)\n",
    "        return {'input_ids': input_ids, 'masks': masks,\\\n",
    "                'labels': labels}\n",
    "    \n",
    "# 准备数据\n",
    "train_dataset = SeqLabelDataset(train_X, train_Y)\n",
    "test_dataset = SeqLabelDataset(test_X, test_Y)\n",
    "\n",
    "from tqdm import tqdm, trange\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def train(model, batch_size, epochs, learning_rate):\n",
    "    train_dataloader = DataLoader(train_dataset,\n",
    "        batch_size=batch_size, shuffle=True, \n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    optimizer = Adam(model.parameters(), lr=learning_rate)\n",
    "    model.zero_grad()\n",
    "    tr_step = []\n",
    "    tr_loss = []\n",
    "    global_step = 0\n",
    "    \n",
    "    with trange(epochs, desc='epoch', ncols=60) as pbar:\n",
    "        for epoch in pbar:\n",
    "            model.train()\n",
    "            for step, batch in enumerate(train_dataloader):\n",
    "                global_step += 1\n",
    "                loss = model(**batch)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                model.zero_grad()\n",
    "                pbar.set_description(f'epoch-{epoch}, '+\\\n",
    "                    f'loss={loss.item():.2f}')\n",
    "                \n",
    "                if epoch > 0:\n",
    "                    tr_step.append(global_step)\n",
    "                    tr_loss.append(loss.item())\n",
    "\n",
    "    # 打印损失曲线\n",
    "    plt.plot(tr_step, tr_loss, label='train loss')\n",
    "    plt.xlabel('training steps')\n",
    "    plt.ylabel('loss')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "                \n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_tags = len(dataset.label2id)\n",
    "crf = CRF(vocab_size, hidden_size, n_tags)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "train(crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4a1e7e25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.5474937690390473, recall = 0.23569384835479257, f1 = 0.3295274606217185\n",
      "precision = 0.4819277108433735, recall = 0.19595835884874463, f1 = 0.2786242925555072\n"
     ]
    }
   ],
   "source": [
    "# 验证效果\n",
    "\n",
    "def evaluate(X, Y, model, batch_size):\n",
    "    dataset = SeqLabelDataset(X, Y)\n",
    "    dataloader = DataLoader(dataset, batch_size=batch_size,\\\n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        P = []\n",
    "        for batch in dataloader:\n",
    "            preds = model.decode(batch['input_ids'], batch['masks'])\n",
    "            P.extend(preds)\n",
    "\n",
    "    p, r, f = compute_metric(Y, P)\n",
    "    print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "    \n",
    "evaluate(train_X, train_Y, crf, batch_size)\n",
    "evaluate(test_X, test_Y, crf, batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc369a17",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "这里介绍基于双向长短期记忆-条件随机场（BiLSTM-CRF）结构的代码实现。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "990c5bc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "# 定义模型，将长短期记忆的输出输入给条件随机场，\n",
    "# 利用条件随机场计算损失和解码\n",
    "class LSTM_CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, num_layers,\\\n",
    "                 dropout, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.lstm = nn.LSTM(input_size=hidden_size,\n",
    "                            hidden_size=hidden_size,\n",
    "                            num_layers=num_layers,\n",
    "                            batch_first=False,\n",
    "                            dropout=dropout,\n",
    "                            bidirectional=True)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size * 2)\n",
    "    \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(hidden_states, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 调用条件随机场进行解码\n",
    "        return self.crf.decode(hidden_states, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ab41a9f0",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, loss=32.30: 100%|█| 20/20 [02:27<00:00,  7.39s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABuo0lEQVR4nO3deXiU1d3G8e9kJpnsCUnIBmHfSVgEZFVAWRVQscVdadXWKijutdoWbRW1VbTQ0tb64oKKbQWLG5vIJiLIImHfAgRICEvIvs487x+TmWRCAiHbZLk/1zVXmXnOzJwHCrk953fOMRmGYSAiIiLSRHl5ugMiIiIidUlhR0RERJo0hR0RERFp0hR2REREpElT2BEREZEmTWFHREREmjSFHREREWnSLJ7uQENgt9s5efIkQUFBmEwmT3dHREREqsAwDLKysoiNjcXLq/LxG4Ud4OTJk8TFxXm6GyIiIlINycnJtG7dutLrCjtAUFAQ4PjNCg4O9nBvREREpCoyMzOJi4tz/RyvjMIOuKaugoODFXZEREQamUuVoKhAWURERJo0hR0RERFp0hR2REREpEnzaM3OvHnzmDdvHkeOHAGgZ8+e/O53v2P8+PEATJ06lXfffdftPQMHDmTjxo2u5wUFBTzxxBN89NFH5OXlce211/K3v/3tolXZIiLSvNhsNoqKijzdDblM3t7emM3mGn+OR8NO69atefnll+nUqRMA7777LjfccAPbtm2jZ8+eAIwbN4758+e73uPj4+P2GTNmzOCzzz5j4cKFhIeH8/jjjzNhwgS2bNlSK79BIiLSeBmGQWpqKufPn/d0V6SaQkNDiY6OrtE+eB4NOxMnTnR7/uKLLzJv3jw2btzoCjtWq5Xo6OgK35+RkcHbb7/N+++/z6hRowBYsGABcXFxrFy5krFjx9btDYiISIPmDDqRkZH4+/tr49hGxDAMcnNzSUtLAyAmJqban9Vglp7bbDb+85//kJOTw+DBg12vr169msjISEJDQxk+fDgvvvgikZGRAGzZsoWioiLGjBnjah8bG0t8fDwbNmyoNOwUFBRQUFDgep6ZmVlHdyUiIp5is9lcQSc8PNzT3ZFq8PPzAyAtLY3IyMhqz9h4vEA5MTGRwMBArFYrDzzwAIsXL6ZHjx4AjB8/ng8++IBVq1bx2muvsXnzZq655hpXUElNTcXHx4cWLVq4fWZUVBSpqamVfuesWbMICQlxPbR7sohI0+Os0fH39/dwT6QmnH9+Nam58vjITteuXdm+fTvnz5/nk08+4Z577mHNmjX06NGDW265xdUuPj6e/v3707ZtW7744gsmT55c6WcahnHRocpnnnmGxx57zPXcuQOjiIg0PZq6atxq48/P42HHx8fHVaDcv39/Nm/ezJtvvsk//vGPC9rGxMTQtm1bDhw4AEB0dDSFhYWkp6e7je6kpaUxZMiQSr/TarVitVpr+U5ERESkIfL4NFZ5hmG41dOUdfbsWZKTk11FSv369cPb25sVK1a42qSkpLBz586Lhh0RERFpPjwadn7zm9+wbt06jhw5QmJiIs8++yyrV6/mjjvuIDs7myeeeILvvvuOI0eOsHr1aiZOnEhERAQ33XQTACEhIdx77708/vjjfP3112zbto0777yThIQE1+osERGR5q5du3a88cYbHv8MT/HoNNapU6e46667SElJISQkhF69erF06VJGjx5NXl4eiYmJvPfee5w/f56YmBhGjhzJxx9/7Ha66ezZs7FYLEyZMsW1qeA777zTqPbYySu04efTePorIiJ1a8SIEfTp06fWwsXmzZsJCAiolc9qjDwadt5+++1Kr/n5+bFs2bJLfoavry9z5sxhzpw5tdm1evN/65N48cs9vPuzKxnWOcLT3RERkUbCMAxsNhsWy6V/lLds2bIeetRwNbianeZmW/J5bHaDbcfSPd0VEZEmzzAMcguLPfIwDKNKfZw6dSpr1qzhzTffxGQyYTKZXOUcJpOJZcuW0b9/f6xWK+vWrePQoUPccMMNREVFERgYyIABA1i5cqXbZ5afgjKZTPzrX//ipptuwt/fn86dO7NkyZLL+r08duwYN9xwA4GBgQQHBzNlyhROnTrluv7jjz8ycuRIgoKCCA4Opl+/fvzwww8AHD16lIkTJ9KiRQsCAgLo2bMnX3755WV9/+Xw+Gqs5q6gyAZAVkGxh3siItL05RXZ6PG7S88a1IXdL4zF3+fSP3bffPNN9u/fT3x8PC+88ALgGJlxniP51FNP8ec//5kOHToQGhrK8ePHue666/jjH/+Ir68v7777LhMnTmTfvn20adOm0u95/vnnefXVV/nTn/7EnDlzuOOOOzh69ChhYWGX7KNhGNx4440EBASwZs0aiouLefDBB7nllltYvXo1AHfccQd9+/Zl3rx5mM1mtm/fjre3NwAPPfQQhYWFrF27loCAAHbv3k1gYOAlv7e6FHY8rNBmByAzTwfUiYiIY/GNj48P/v7+FR6X9MILLzB69GjX8/DwcHr37u16/sc//pHFixezZMkSpk2bVun3TJ06ldtuuw2Al156iTlz5rBp0ybGjRt3yT6uXLmSHTt2kJSU5Nqn7v3336dnz55s3ryZAQMGcOzYMZ588km6desGQOfOnV3vP3bsGDfffDMJCQkAdOjQ4ZLfWRMKOx5WUOQIO1n5GtkREalrft5mdr/gmXMT/bxrZyFK//793Z7n5OTw/PPP8/nnn3Py5EmKi4vJy8vj2LFjF/2cXr16uX4dEBBAUFCQ6xyqS9mzZw9xcXFuG/L26NGD0NBQ9uzZw4ABA3jssce47777XOdX/vSnP6Vjx44APPzww/zqV79i+fLljBo1iptvvtmtP7VNNTse5hrZydfIjohIXTOZTPj7WDzyqK2dnMuvqnryySf55JNPePHFF1m3bh3bt28nISGBwsLCi36Oc0qp7O+N3W6vUh8qO6mg7OszZ85k165dXH/99axatYoePXqwePFiAO677z4OHz7MXXfdRWJiIv3796/ThUYKOx5WUOyo2dE0loiIOPn4+GCz2arUdt26dUydOpWbbrqJhIQEoqOjXfU9daVHjx4cO3aM5ORk12u7d+8mIyOD7t27u17r0qULjz76KMuXL2fy5MnMnz/fdS0uLo4HHniARYsW8fjjj/PWW2/VWX8VdjyssFjTWCIi4q5du3Z8//33HDlyhDNnzlx0xKVTp04sWrSI7du38+OPP3L77bdXeYSmukaNGkWvXr2444472Lp1K5s2beLuu+9m+PDh9O/fn7y8PKZNm8bq1as5evQo3377LZs3b3YFoRkzZrBs2TKSkpLYunUrq1atcgtJtU1hx8MKijWNJSIi7p544gnMZjM9evSgZcuWF62/mT17Ni1atGDIkCFMnDiRsWPHcsUVV9Rp/0wmE59++iktWrTg6quvZtSoUXTo0IGPP/4YALPZzNmzZ7n77rvp0qULU6ZMYfz48Tz//PMA2Gw2HnroIbp37864cePo2rUrf/vb3+quv0ZVF/43YZmZmYSEhJCRkUFwcHC9fvfgWV+TkpGPj8WL/X8cX6/fLSLSlOXn55OUlET79u3x9fX1dHekmi7251jVn98a2fEw58hOYbGd/KKqzc+KiIhI1SnseJizZgc0lSUiIlIXFHY8zLkaC1SkLCIiUhcUdjzIbjcospWWTGn5uYhI7VNpauNWG39+Cjse5NxQ0EkjOyIitce5aV5ubq6HeyI14fzzK78J4uXQcREeVFDsHnZUsyMiUnvMZjOhoaGuIxD8/f1rbRdjqXuGYZCbm0taWhqhoaGYzdU/bkNhx4PK1usAZOZpZEdEpDY5D9Ks6plP0vCEhoZWeCDq5VDY8aDC4vLTWBrZERGpTSaTiZiYGCIjIykq0r+xjY23t3eNRnScFHY8SNNYIiL1w2w218oPTWmcVKDsQReO7GgaS0REpLYp7HjQBSM7WnouIiJS6xR2PKj8yE6mRnZERERqncKOB5VfjaUCZRERkdqnsONBF4zsaOm5iIhIrVPY8SBnzY7V4vhj0MiOiIhI7VPY8SDnyE5EoBVQzY6IiEhdUNjxIFfYCXKEneyCYmx2HVgnIiJSmxR2PMhZoNwy0Mf1WrZGd0RERGqVwo4HOWt2AqwWfL0dfxTaRVlERKR2Kex4UNkC5SBfx9H1zrDz783JDJn1NbtPZnqsfyIiIk2Bwo4HOWt2fCxeBPs6jilzLj9fvO0EJzPy+e7wWY/1T0REpClQ2PGg0pEdM8F+jpEd5/Lzw2eygQv34hEREZHLo7DjQWVHdkqnsYrJKSjmVGYBcOEuyyIiInJ5FHY8yBlkrG7TWEUknclxtckv0siOiIhITSjseFBFIztZ+cVuYUcjOyIiIjWjsONB7jU7JSM7+UUcPp1zQRsRERGpHounO9Ccua/GKi1QPptd4GpToGksERGRGlHY8aCKa3aKScnIu6CNiIiIVI/CjgcV2ko3FfQxl+6gfPiMprFERERqi8KOBzmnqKwWL3wsjrCTdCaHrDLnYynsiIiI1IwKlD3IObJTdjVWSka+W5uCIk1jiYiI1ITCjgeVjuyYXQXKTl6mkjYa2REREakRhR0PKjuy41x67tQuIgBQ2BEREakphR0Pck5RlT313Kl7TLCjjVZjiYiI1IhHw868efPo1asXwcHBBAcHM3jwYL766ivXdcMwmDlzJrGxsfj5+TFixAh27drl9hkFBQVMnz6diIgIAgICmDRpEsePH6/vW6mWsiM7AT5m19QVQPfoIED77IiIiNSUR8NO69atefnll/nhhx/44YcfuOaaa7jhhhtcgebVV1/l9ddfZ+7cuWzevJno6GhGjx5NVlaW6zNmzJjB4sWLWbhwIevXryc7O5sJEyZgszWsEZGvElMY+vIqthxNd71WtmbHZDK5je6Ujuwo7IiIiNSER8POxIkTue666+jSpQtdunThxRdfJDAwkI0bN2IYBm+88QbPPvsskydPJj4+nnfffZfc3Fw+/PBDADIyMnj77bd57bXXGDVqFH379mXBggUkJiaycuVKT97aBVbsOcWJ83ms2X/a9VpBmZEdwFW3Y7V40TbcWbPTsEKbiIhIY9NganZsNhsLFy4kJyeHwYMHk5SURGpqKmPGjHG1sVqtDB8+nA0bNgCwZcsWioqK3NrExsYSHx/valORgoICMjMz3R51La/QEVpyChx76BiG4TouwloSdoKsjpGd9hEB+PmYHX3VyI6IiEiNeDzsJCYmEhgYiNVq5YEHHmDx4sX06NGD1NRUAKKiotzaR0VFua6lpqbi4+NDixYtKm1TkVmzZhESEuJ6xMXF1fJdXSivyD3sOOt14MKRnQ4tA1wBqLDYjmEYdd4/ERGRpsrjYadr165s376djRs38qtf/Yp77rmH3bt3u66bTCa39oZhXPBaeZdq88wzz5CRkeF6JCcn1+wmqiC3ZGQnqyTslB2xcY3s+JaO7DhfK99WRERELo/Hw46Pjw+dOnWif//+zJo1i969e/Pmm28SHR0NcMEITVpammu0Jzo6msLCQtLT0yttUxGr1epaAeZ81LX88iM7ZQKM81ysq7u0JMDHzMiukVgtZtd1hR0REZHq83jYKc8wDAoKCmjfvj3R0dGsWLHCda2wsJA1a9YwZMgQAPr164e3t7dbm5SUFHbu3Olq01DklqvZcQYYH4uXaxTqrkFtSZw5lv7twvA2mzC5dlFWkbKIiEh1efQg0N/85jeMHz+euLg4srKyWLhwIatXr2bp0qWYTCZmzJjBSy+9ROfOnencuTMvvfQS/v7+3H777QCEhIRw77338vjjjxMeHk5YWBhPPPEECQkJjBo1ypO3dgFngbLzkE9XcbLZPW96lWy2YzKZsFq8yC+ya68dERGRGvBo2Dl16hR33XUXKSkphISE0KtXL5YuXcro0aMBeOqpp8jLy+PBBx8kPT2dgQMHsnz5coKCglyfMXv2bCwWC1OmTCEvL49rr72Wd955B7PZXNnXeoSrQLnQObLjeO5jqXxwzWoxO8KORnZERESqzaNh5+23377odZPJxMyZM5k5c2albXx9fZkzZw5z5syp5d7VrtySkJNT4Agu5ZedV8R5LV8jOyIiItXW4Gp2miK73XAFluz8C2t2KmP19nJrKyIiIpdPYace5JeZhiq0OaalSkd2Kp9uc17TNJaIiEj1KezUA+dKLKecAlsVa3Y0siMiIlJTCjv1IO+CsFN8WTU7Wo0lIiJSfQo79cC5Esspu6C4ajU7msYSERGpMYWdelB+ZKds2LnoyI4KlEVERGpMYacelK/ZqerIjq9FJ5+LiIjUlMJOPcgrKnZ77l6zc5HVWM6RnSJNY4mIiFSXwk49yCt0H5nJzi/WaiwREZF6orBTD5y7JztlV3k1lqaxREREakphpx5UfzWWc2RH01giIiLV5dGzsZqLivbZsRuOX1etZkcjOyIiItWlsFMPLlyNZcPiZQKqus+Owo6IiEh1aRqrHuSXTGM5A45jGsvxWpV2UNY0loiISLUp7NQD58hORKAVqMZxERrZERERqTaFnXrgDDstgxxhp+o7KJdMY6lmR0REpNoUduqBcxor0hl28ktHdrQaS0REpG4p7NQD5z47rmmswrIjOxdZjWXRyI6IiEhNKezUA+c+Oy01siMiIlLvFHbqQV6FNTtVWI2lU89FRERqTGGnHpQvUC4otpNTWJWzsbTPjoiISE0p7NSD8tNYAOk5hcClanZ06rmIiEhNKezUA+c0VrCvt2skJz3XEXYuOrKjaSwREZEaU9ipB86RHT9vM0FWxwkdpWdjaRpLRESkLins1DHDMErDjo+ZAKv7cWRajSUiIlK3FHbqWH6RHaNkFKeisHOxkR3fkh2Ui2wGNudQkIiIiFwWhZ06llemuLjsNJZTVUZ2ANe+PCIiInJ5FHbqmHP3ZKvFC7OXiQCr++qrqqzGAk1liYiIVJfCTh1zrsTy83GEmkBfb7frF5vGspgdAQlUpCwiIlJdCjt1zDmN5V9SfxNYbmTHx3zxP4LSvXYUdkRERKpDYaeOOXdP9i0Z2QnwKa3Z8Tab8CoZuamMVmSJiIjUjMJOHXON7LimsUrDzsXqdcq30TSWiIhI9Sjs1DFnzY6/tyPkBJZZjXWxlVhOpbsoa2RHRESkOhR26lj5aayyYedixcnl26hmR0REpHoUdupY+QLlgMsd2dE0loiISI0o7NSxvJJ9dvxqOrKjaSwREZFqUdipY3mFjhEZvwoKlC+vZkcjOyIiItWhsFPHcoscIzuuaawyS88vtccOlE5j5RdpZEdERKQ6FHbqWPkdlIMue+m5RnZERERqQmGnjpUPO5dfoKzVWCIiIjWhsFPHci9YjVU6mlO1AmXnaixNY4mIiFSHwk4dyy83smO1mF21OipQFhERqXsKO3Us1xV2SqevnKM7qtkRERGpex4NO7NmzWLAgAEEBQURGRnJjTfeyL59+9zaTJ06FZPJ5PYYNGiQW5uCggKmT59OREQEAQEBTJo0iePHj9fnrVTKOY3l510abJzLz6sysuNb8r4CrcYSERGpFo+GnTVr1vDQQw+xceNGVqxYQXFxMWPGjCEnJ8et3bhx40hJSXE9vvzyS7frM2bMYPHixSxcuJD169eTnZ3NhAkTsNk8HxCc01jOg0ChdPn55W0qqJEdERGR6rBcukndWbp0qdvz+fPnExkZyZYtW7j66qtdr1utVqKjoyv8jIyMDN5++23ef/99Ro0aBcCCBQuIi4tj5cqVjB07tu5uoAqc++z4lQk7zl2UL69AWWFHRESkOhpUzU5GRgYAYWFhbq+vXr2ayMhIunTpwv33309aWprr2pYtWygqKmLMmDGu12JjY4mPj2fDhg0Vfk9BQQGZmZluj7ri2kG5gmmsKoUdnXouIiJSIw0m7BiGwWOPPcawYcOIj493vT5+/Hg++OADVq1axWuvvcbmzZu55pprKCgoACA1NRUfHx9atGjh9nlRUVGkpqZW+F2zZs0iJCTE9YiLi6uz+3KejeU2jWWtes2O9tkRERGpmQYTdqZNm8aOHTv46KOP3F6/5ZZbuP7664mPj2fixIl89dVX7N+/ny+++OKin2cYBiaTqcJrzzzzDBkZGa5HcnJyrd1H+T5UVKDcu3UIAN1jgi/5GZrGEhERqRmP1uw4TZ8+nSVLlrB27Vpat2590bYxMTG0bduWAwcOABAdHU1hYSHp6eluoztpaWkMGTKkws+wWq1Yrdbau4FKFBTbMQzHr8vW7Pzi6o5M6R9HqL/PJT9Dp56LiIjUjEdHdgzDYNq0aSxatIhVq1bRvn37S77n7NmzJCcnExMTA0C/fv3w9vZmxYoVrjYpKSns3Lmz0rBTX5xHRYD7yA5QpaAD2lRQRESkpjw6svPQQw/x4Ycf8r///Y+goCBXjU1ISAh+fn5kZ2czc+ZMbr75ZmJiYjhy5Ai/+c1viIiI4KabbnK1vffee3n88ccJDw8nLCyMJ554goSEBNfqLE9xTmH5mL2wVOGE84q4prFUsyMiIlItHg078+bNA2DEiBFur8+fP5+pU6diNptJTEzkvffe4/z588TExDBy5Eg+/vhjgoKCXO1nz56NxWJhypQp5OXlce211/LOO+9gNl96h+K6VP4Q0OrQNJaIiEjNeDTsGM6Clkr4+fmxbNmyS36Or68vc+bMYc6cObXVtVrhCjveNQk7KlAWERGpiQazGqspyiu6cPfky6WaHRERkZpR2KlDuYUX7p58uUr32dE0loiISHUo7NQhTWOJiIh4nsJOHXJOY9XGyE6x3aDYpsAjIiJyuRR26lBubYzseJf+EWl0R0RE5PIp7NSh/FooUPYxK+yIiIjUhMJOHXKN7PhUf4W/xeyFxctxxpf22hEREbl8Cjt1qDamsQB8vbWLsoiISHUp7NSh2pjGgrK7KCvsiIiIXC6FnTpUG/vsgI6MEBERqQmPHhfR1D02uit3DWpHyyBrjT7H6q29dkRERKpLYacORYf4Eh3iW+PPKd1FWWFHRETkcmkaqxHQNJaIiEj1Kew0AjoyQkREpPoUdhqB0pPPNbIjIiJyuRR2GgHnNFa+anZEREQum8JOIxDs6w1Aem6hh3siIiLS+CjsNALOFV2nMvI93BMREZHGR2GnEXCGnRSFHRERkcumsNMIRAWXjOxkKuyIiIhcLoWdRiCmZGQnVWFHRETksinsNALRJSM7p7MKKLZpRZaIiMjlUNhpBMIDrVi8TNgNOJ1d4OnuiIiINCoKO42A2ctEZMlhoqkqUhYREbksCjuNRJSzbkdhR0RE5LIo7DQSKlIWERGpHoWdRsK5/FwjOyIiIpdHYaeR0MiOiIhI9SjsNBIa2REREakehZ1GwrnXjkZ2RERELo/CTiMRE+IHOEZ2DMPwcG9EREQaD4WdRiIy2LHPTkGxnYy8Ig/3RkREpPFQ2GkkfL3NhAX4ADr9XERE5HIo7DQiUarbERERuWwKO41IjHZRFhERuWwKO41IVZeff5WYwoMfbCEjV7U9IiIiCjuNiHP5+amLTGNtOXqOhxdu48vEVD7dfqK+uiYiItJgKew0Is5prMoKlNOy8vnVgq0U2RxL0xNPZNRb30RERBoqhZ1GxHnyeUUjO0U2O9M+2EZaVgH+PmYAEo8r7IiIiCjsNCIXG9mZs+ogm46cI8hq4V/39AfgQFoWuYXF9dpHERGRhkZhpxFxFihn5BWRX2Rzu7ZsZyoAv5vYgyEdI4gMsmI3YE9KZr33U0REpCFR2GlEgn0t+Hk7pqjKrsjKK7RxIC0LgKu7tAQgoVUIADs0lSUiIs2cwk4jYjKZKpzK2puaid2AiEArkUGOYyUSWjvCjup2RESkufNo2Jk1axYDBgwgKCiIyMhIbrzxRvbt2+fWxjAMZs6cSWxsLH5+fowYMYJdu3a5tSkoKGD69OlEREQQEBDApEmTOH78eH3eSr1xTmWlZOS5Xtt50jFVFd8qGJPJBECvkrCzQyuyRESkmfNo2FmzZg0PPfQQGzduZMWKFRQXFzNmzBhycnJcbV599VVef/115s6dy+bNm4mOjmb06NFkZWW52syYMYPFixezcOFC1q9fT3Z2NhMmTMBms1X0tY1a1+ggALYeS3e9tqsk0MTHhrheiy+Zxjp0OpucAhUpi4hI8+XRsLN06VKmTp1Kz5496d27N/Pnz+fYsWNs2bIFcIzqvPHGGzz77LNMnjyZ+Ph43n33XXJzc/nwww8ByMjI4O233+a1115j1KhR9O3blwULFpCYmMjKlSs9eXt1YnDHcAA2HDrrem3nyZKw0yrY9VpkkC8xIb4YBuw6qSJlERFpvhpUzU5GhuOHdlhYGABJSUmkpqYyZswYVxur1crw4cPZsGEDAFu2bKGoqMitTWxsLPHx8a425RUUFJCZmen2aCwGdQjHywSHT+eQmpFPYbGdfamOUa6eZUZ2oHR0Z8fx8/XdTRERkQajwYQdwzB47LHHGDZsGPHx8QCkpjqWU0dFRbm1jYqKcl1LTU3Fx8eHFi1aVNqmvFmzZhESEuJ6xMXF1fbt1JkQP2/XSqtvD55h/6ksimwGIX7etG7h59a2V0k77aQsIiLNWYMJO9OmTWPHjh189NFHF1xzFt06GYZxwWvlXazNM888Q0ZGhuuRnJxc/Y57wOCOEYBjKmtXmSms8verFVkiIiINJOxMnz6dJUuW8M0339C6dWvX69HR0QAXjNCkpaW5Rnuio6MpLCwkPT290jblWa1WgoOD3R6NydBOzrqdM65Rm/hyU1hQutfO4TM5ZOXrBHQREWmePBp2DMNg2rRpLFq0iFWrVtG+fXu36+3btyc6OpoVK1a4XissLGTNmjUMGTIEgH79+uHt7e3WJiUlhZ07d7raNDX924bhY/YiJSOfZbtOAdCz1YVhJzzQSqtQx9TW3FUHL9h1WUREpDmoVth59913+eKLL1zPn3rqKUJDQxkyZAhHjx6t8uc89NBDLFiwgA8//JCgoCBSU1NJTU0lL8+xh4zJZGLGjBm89NJLLF68mJ07dzJ16lT8/f25/fbbAQgJCeHee+/l8ccf5+uvv2bbtm3ceeedJCQkMGrUqOrcXoPn52PmirahAJzOKgAgPrbi0akb+8YC8I+1hxn1+hqW7aq4jklERKSpqlbYeemll/Dzc4wYfPfdd8ydO5dXX32ViIgIHn300Sp/zrx588jIyGDEiBHExMS4Hh9//LGrzVNPPcWMGTN48MEH6d+/PydOnGD58uUEBQW52syePZsbb7yRKVOmMHToUPz9/fnss88wm83Vub1GYUhJ3Q5AgI+ZduEBFbZ7YkxX3ry1D9HBvhxPz+OX729hy9H0CtuKiIg0RSbDMIzLfZO/vz979+6lTZs2PP3006SkpPDee++xa9cuRowYwenTp+uir3UmMzOTkJAQMjIyGk39zpaj57h53ncAXNkujH8/MPii7XMLi/nFe1tYf/AMj47qwiOjOtdHN0VEROpMVX9+V2tkJzAwkLNnHZvaLV++3DVd5Ovr65qCkrrVq3UoAT6OkauerS4d0Px9LIzsFgloKbqIiDQvluq8afTo0dx333307duX/fv3c/311wOwa9cu2rVrV5v9k0p4m70Y0S2SL3akMLB9eJXe41ydtVNhR0REmpFqjez89a9/ZfDgwZw+fZpPPvmE8HDHD9stW7Zw22231WoHpXIv3ZjAB/cNZGzPipfYl9czNhiTCVIz812FzSIiIk1dtWp2mprGWLNTXde+tppDp3OY/7MBjOwa6enuiIiIVFud1uwsXbqU9evXu57/9a9/pU+fPtx+++0XbO4nDYtrKku7KouISDNRrbDz5JNPug7PTExM5PHHH+e6667j8OHDPPbYY7XaQald8ZdxXtacrw9ww1+/5eR5FZ2LiEjjVa0C5aSkJHr06AHAJ598woQJE3jppZfYunUr1113Xa12UGpXfBWLlE9nFfCXVQcoshk8syiRd3424JLnkYmIiDRE1RrZ8fHxITc3F4CVK1cyZswYAMLCwlwjPtIw9SzZaflkRj5nsysvUv548zGKbI5yrjX7T7No64l66Z+IiEhtq1bYGTZsGI899hh/+MMf2LRpk2vp+f79+90O8pSGJ8jXmw4Rjt2WK5vKKrbZ+fD7YwD0a9sCgBc+301aVn79dFJERKQWVSvszJ07F4vFwn//+1/mzZtHq1atAPjqq68YN25crXZQap9zKmvXyYpH4b7em8bJjHzCAnx47+dXEt8qmIy8In7/v1312U0REZFaUa2anTZt2vD5559f8Prs2bNr3CGpewmtQljy40kSK1mR9f53jsNcp/SPI8Bq4dWbezNp7nq+2pnKjuPn6dU6tB57KyIiUjPVCjsANpuNTz/9lD179mAymejevTs33HBDkz58s6m42IqsQ6ezWX/wDCYT3DGwDQA9YoMZ1jmC1ftOs+N4hsKOiIg0KtUKOwcPHuS6667jxIkTdO3aFcMw2L9/P3FxcXzxxRd07Nixtvsptch5ltaJ83mk5xTSIsDHdW3BRseozjVdI4kL83e93jU6iNX7TrP/VFb9dlZERKSGqlWz8/DDD9OxY0eSk5PZunUr27Zt49ixY7Rv356HH364tvsotSzY15v2lRQpr9qbBsCtV7Zxe71rVBAA+1IVdkREpHGp1sjOmjVr2LhxI2FhYa7XwsPDefnllxk6dGitdU7qTo/YYJLO5LDrZCZXd2kJQFZ+EUfPOrYUcK7CcupSEnb2n8rCMAztuSMiIo1GtUZ2rFYrWVkX/hd+dnY2Pj4+FbxDGhrnfju7TpaO7DhHbaKDfQkLcP9z7BQZiJcJ0nOLOH2R/XlEREQammqFnQkTJvCLX/yC77//HsMwMAyDjRs38sADDzBp0qTa7qPUgZ6xjiLl3WWWn+9Ocfy6e0zQBe19vc20C3dMfe1Pza6HHoqIiNSOaoWdv/zlL3Ts2JHBgwfj6+uLr68vQ4YMoVOnTrzxxhu13EWpC86RnaSzOeQUFAOwpyTs9Iit+ORY51TWPhUpi4hII1Ktmp3Q0FD+97//cfDgQfbs2YNhGPTo0YNOnTrVdv+kjkQEWokKtnIqs4C9qZn0axvmGuXpHlNJ2IkOYumuVPZfZpHyzhMZJJ/LZXxCTI37LSIicrmqHHYudZr56tWrXb9+/fXXq90hqT89Y0M4lZnGrpOZ9Ilr4Rqx6VFJ2OkW7RjZ2XuZIzsPfbiVo2dzWfnY1XSKvHCKTEREpC5VOexs27atSu20Sqfx6BkbzKq9aew6kUnSmRzyi+z4eZtpW1KbU55zGuvAqSzsdgMvr0v/WRfb7CSfc6zw2p2SpbAjIiL1rsph55tvvqnLfogHOEdwdqVkuIqTu0YHYa4kxLQL98fH7EVuoY0T5/PcNh0EeGXpXr49eIaP7h9EgNXxf62zOYXYHYencyhNhc0iIlL/qlWgLE2Dc0XW/tRsEo+fByovTgawmL3oGBkIXLi5oGEYLPjuKDuOZ7A9+bzr9bTM0mXqh04r7IiISP1T2GnG4sL8CPK1UGiz8/mOFKDy4mSnrlElYadc3c65nEKySlZ1pWTku14/lVn660Onc2ql3yIiIpdDYacZM5lMrqksZ0CprDjZqUt06U7KZR05WxpkygactKzSkZ3Dp7OxO+e0RERE6onCTjPnnMoCMJlKV1xVprIzspLO5Lp+nZKR5/p1WlZp8CkotnPifB4iIiL1SWGnmetZpkanXXiAq7C4Ms4VWYdP51Bks7teP3KmdGQn1W0ay/1oCdXtiIhIfVPYaeZ6tioNOxUdE1Fe6xZ+BPiYKbTZ3QJO2WmssjU7p8uM7IDqdkREpP4p7DRzHVsG4mNx/N/gUvU64Kjz6eZcsl7mXK2yYafsyI6zZsdZ+KyRHRERqW8KO82ct9mLPnGhAFzRtkWV3pPQylHnk3jCcWK6YRgcKVOzczankIJiG1BarDy4QzigvXZERKT+KewIr/20N/+6uz9DOkZUqX18ubBzNqeQ7IJiTCbwMTv+L5WWWYDNbnAmuxCAIR1Lwo6msUREpJ5V6yBQaVriwvwv2A35YpwjO7tOZGC3G67andgQPyxmE0fP5pKSkY/V2wub3cBkggHtwwA4k11ARm4RIf7ebp+54/h5zmQXcE23qFq6KxEREQeN7Mhl69gyAD9vMzmFNg6fySGpJOy0jwggOtgXcCw/d+6eHB5gJcTPm5gQx7VDZ9ynsnILi7njre/5+Ts/8M3etHq8ExERaQ4UduSyWcxermMldp7I4OhZR71O23B/oksCTWpGPqdLipOjgq2AoxgaLqzb+Sox1bX78u+W7CSv0Fb3NyEiIs2Gwo5Ui3Mqa8fxDJLOlhnZcYadzHxXcXJkkDPsOE5TL1+3898tx12/Tj6Xx9xvDtRt50VEpFlR2JFqcRYp7zyR4arZaRceQExw6chOmmtkx/Ga8xDRssvPj6fn8t3hs5hM8LsJPQD459rDHCh3HIWIiEh1KexItfRqXVKkfLJM2IkIIDrED3BsLOg8KqJ0ZOfCaaxFW08AjtVaPxvajlHdIymyGTz76U4MQ+doiYhIzSnsSLV0bBnoKlLOKbThZXKcol62Zsd5VERL58hOSdg5ei6XwmI7hmG4prB+0q81JpOJmZN6YrV4sSnpnNumhSIiItWlsCPVYvYyuYqUAWJD/bBazK4VV6ezC1wHgkaVjOxEBVsJ8rVgsxvc994PLNp6gmPncgm0WhjbMxqA1i38uapzSwDW7D9dn7ckIiJNlMKOVJuzSBkcxckAEYFWzF4mbHaD/amO6arIkpEdk8nEbyf0wMfixdr9p3n8Pz8CcH1CDP4+pVs+De/i2NxwrcKOiIjUAoUdqbayYadtuGNTQrOXyTWSU1hyKrpz6TnAlP5xLH3kKteOygA/6d/a7XOv7uIY2dlyNJ3skiXpIiIi1aUdlKXaElqXhp124QGuX0eH+HKy5DBQk8kx2lNWh5aBfHDfQJbtSqWg2M6AdmFu19uGB9A23J+jZ3P57tBZRvfQrsoiIlJ9GtmRanMWKUPpNBbgKlIGCA/wwdt84f/NTCYT4+JjuKFPqwo/++qSuh1NZYmISE15NOysXbuWiRMnEhsbi8lk4tNPP3W7PnXqVEwmk9tj0KBBbm0KCgqYPn06ERERBAQEMGnSJI4fP47UPbOXiZ8Pa0fvuFCubF86OhMd7Of6dcsg34reeknOqay1BxR2RESkZjwadnJycujduzdz586ttM24ceNISUlxPb788ku36zNmzGDx4sUsXLiQ9evXk52dzYQJE7DZdORAfXhybDf+99BQgnxLD/aMKTOy49xj53IN7hiOxctxqOjRszopXUREqs+jNTvjx49n/PjxF21jtVqJjo6u8FpGRgZvv/0277//PqNGjQJgwYIFxMXFsXLlSsaOHVvrfZZLKzuNVbY4+XIEWi30a9uC75POsXb/ae4aHHDpN4mIiFSgwdfsrF69msjISLp06cL9999PWlrpqdhbtmyhqKiIMWPGuF6LjY0lPj6eDRs2VPqZBQUFZGZmuj2k9kS7jexUbxoLSqeytN+OiIjURIMOO+PHj+eDDz5g1apVvPbaa2zevJlrrrmGggLHzrypqan4+PjQokULt/dFRUWRmppa6efOmjWLkJAQ1yMuLq5O76O5iQ6u+cgOwPCSsLPh0FkKijUtKSIi1dOgw84tt9zC9ddfT3x8PBMnTuSrr75i//79fPHFFxd9n2EYmEymSq8/88wzZGRkuB7Jycm13fVmLapM2KlugTJAj5hgooN9yS208WViSm10TUREmqEGHXbKi4mJoW3bthw4cACA6OhoCgsLSU9Pd2uXlpZGVFTle7NYrVaCg4PdHlJ7fCxerhGd2NDqhx0vLxN3DmoDwNvrk3QwqIiIVEujCjtnz54lOTmZmJgYAPr164e3tzcrVqxwtUlJSWHnzp0MGTLEU90U4IUb4pl+TSe3XZar4/aBbbFavNh5IpNNSedqqXciItKceHQ1VnZ2NgcPHnQ9T0pKYvv27YSFhREWFsbMmTO5+eabiYmJ4ciRI/zmN78hIiKCm266CYCQkBDuvfdeHn/8ccLDwwkLC+OJJ54gISHBtTpLPGNsz2jX4Z41ERbgw+QrWvPRpmO8vT6JgR3CL/0mERGRMjwadn744QdGjhzpev7YY48BcM899zBv3jwSExN57733OH/+PDExMYwcOZKPP/6YoKAg13tmz56NxWJhypQp5OXlce211/LOO+9gNpvr/X6kbtw7rB0fbTrGij2nOHo2h7bhWoYuIiJVZzJUCEFmZiYhISFkZGSofqeBuuf/NrFm/2mmDmnHzEk9Pd0dERFpAKr687tR1exI83XvsPYA/OeHZNJzCj3cGxERaUwUdqRRuKpzBD1igskptPHXbw5e+g0iIiIlFHakUTCZTDw9vhsA7313lORzuRdtbxgGK3efYp0OEhURafYUdqTRuLpzBEM7hVNos/Pa8n2VtjtwKovb3trIfe/9wL3v/EBWflE99lJERBoahR1pNEwmE78e1x2AT7efZOeJDLfrdrvB68v3Mf7NdWw87NiTp9Bm50Batlu7dQdO892hs/XTaRER8TiFHWlUElqHcEOfWAD+8PlucguLASi22Xnqkx38ZdVBiu0Go7pH0au1Y0PDA6eyXO8/l1PIz9/ZzD3zN5GpER8RkWZBYUcanSfGdMXH7MX3SecY9doaPvvxJA8v3MZ/txzH7GXiTz/pxb/u6U//tmEA7D9VOrKz80QGRTaDwmI7W46mV/YVIiLShCjsSKMTF+bPW/f0p3ULP05m5DP9o218mZiKj9mLv95+BT/t7zjFvktUIAD7y4zs7E7JdP16s46fEBFpFhR2pFEa3qUlKx8bziPXdsbH4oWvtxf/uqc/4+JLj6joEu3Yabts2Nl1skzYOaKwIyLSHHj0uAiRmvD1NvPo6C7cNbgtdsMgMsj9hPXOkY6RnVOZBWTkFRHi583uk6VFzT8mZ5BfZMPXW0eLiIg0ZRrZkUYvItB6QdABCPL1JjbE8fqBU1nkFhZz+EwOAIFWC4U2Oz8mn6/ProqIiAco7EiT1jnKOZWVzZ6ULAwDWgZZGd61JaCpLBGR5kDTWNKkdYkKZM3+0+w/lYWt5MzbHjHBXNkujC92pPB90jmmVfC+tftPsz35PDkFxeQUFtM+IpCb+rYiLMCnfm9ARERqTGFHmrQuUaVFygXFNgB6xgYzoJ1jWfrWo+kU2+xYzKWDnGmZ+fzsnc3Y7IbbZ73y1V7GJ0Tz4IhOdC0pfhYRkYZP01jSpHUpM421u2QlVo/YYLpGBxHsayGn0MaelCy39ySeyMBmN4gItHLfsPY8NLIjCa1CKLTZ+d/2k/zk7xs4p5PXRUQaDYUdadI6lazIOpNd4Fp23jM2BLOXif4lozvfJ7kfHeFsd3XnCJ6b0IMnx3bjs+nDWDJtKJ0jA8nKL2bOqgP1eBciIlITCjvSpAVYLbRu4QdAsd0gwMdM2zB/ANdUVvki5bIjQGX1ah3K7yb2AGDBxqMcO3vxk9dFRKRhUNiRJs85lQXQPSYYLy8TAFe2d4addAyjtD7Huctyjxj3sANwVeeWXNU5giKbwZ8ucvK6iIg0HAo70uSVDTs9y4zWJLQKwc/bzLmcQvaV7LKcmV/EsXOOEZvyIztOvx7fDZMJPvvxpPbpERFpBBR2pMlznpEF7gHGx+LFgJLRnW8POup29pRMYbUK9SPUv+Jl5j1jQ7ipTysAXl22t076LCIitUdhR5o895GdELdrQzuGA7Dh4BmgdAqrewVTWGU9OroL4AhJp7MKaq2vIiJS+xR2pMnrFBlIyyArUcFWOpcZ5QEY2ikCgO+TzlFss5dZsXXxsBMX5u+q6fm2JCiJiEjDpLAjTZ6vt5nPpg1jybRhWC3uh372iAkm1N+b7IJifjyeUelKrIpc3cVx5MTa/acrvG4YBgs2HmX9AYUhERFPUtiRZiE6xJeo4AsPC/XyMjG4g2Mqa83+0xxIcxQqV7QSq7yrOztGhdYeOOO2mstp+e5TPPfpTu59dzPJ57RMXUTEUxR2pNkbUlK389GmYxTZDIJ9S/fmuZh+7Vrg523mTHbBBbswG4bB39ccAqCg2M4fPt9d+x0XEZEqUdiRZm9ISd2Os9C4R2wwJpPpku+zWswM6uBYzbXugPtU1qakc2w7dh4fixcWLxPLd5/im31ptdxzERGpCoUdafY6RAQQXWaKq0dMyEVau3PV7ZQLO/NKRnV+2q81U4e0A+D5Jbtch5GKiEj9UdiRZs9kMjGkU7jr+aVWYpXlDDubk9LJK3QEmT0pmazedxovE/zi6g48MqozLYOsHDmby1trD9du50VE5JIUdkSAoR0jXL+uykospw4RAbQK9aPQZmdjyYGi/ygZ1RmfEEPb8ACCfL157vruAMz95iDH01WsLCJSnxR2RHDst+Nj9iLEz5uOLQMv/YYSJpOJq7s4gtJbaw9zx7828r8fTwLwwNUdXe0m9Y5lYPsw8ovs/PHzPbXbeRERuSiFHREcS9M/+sVAPrx/ID6Wy/trcVVnx1TWhkNn+fbgWQwD7hrUloTWpbU/JpOJF26Ix+xlYumu1Er35hERkdqnsCNSol/bsAuOk6iK4V1a0iculG7RQTw5tivrnhrJH26Mv6Bd1+gg7hncDoCZtVSsvP7AGU5l5tf4c0REmjKTUdFuaM1MZmYmISEhZGRkEBxc9XoNkcuVmV/ENX9ew5nsAqZf04nHRnep0jL3iny67QQzPt5OQqsQlkwbWu3PERFprKr681sjOyL1KNjXm99c1w2AOasOcv97P5CacfkjM4XFdl5bsQ+AxBMZbD6SXqv9FBFpShR2ROrZTX1b8eTYrnibTazck8bo2Wv4MjHlsj5j4eZjJJ/Lcz1/d8ORWu6liEjTobAjUs9MJhMPjezE59OvonfrELLyi5nx8XYOnc6u0vtzC4v5y9cHAbhjYBsAlu5KJSUj72JvExFpthR2RDyka3QQn/xqCFd1jqCw2M5T/92BzX7pErp3NhzhTHYBcWF+/H5iT65sH4bNbvDBxmP10GsRkcZHYUfEgyxmL16+uReBVgtbjqZfcjoqI6+Iv692bFr42Ogu+Fi8+FnJcRQfbTpGfpGOoxARKU9hR8TDWoX68UxJ0fKry/Zy9GxOpW3X7j9NZn4xHVoGMKl3KwBG94giNsSXszmF/PqTHTz5nx/5+TubWbX3VL30X0SkoVPYEWkAbr+yDUM6hpNfZOeX72+p9EiJ3SmZAAzuEI7Zy7HU3GL24o5BbQH4dPtJ/rPlOKv2pvHa8v211r8Nh85wzZ9X8+m2E7X2mSIi9UVhR6QBMJlMvHJzL8IDfNibmsXEOev59uCZC9rtPukIO91j3PeTmDqkHbcPbMMt/eP45dUdADiQlk2xzV7jviWfy+XBD7Zy+EwOn2w9XuPPExGpbwo7Ig1EXJg/S6YPI6FVCOm5Rdz19vf854dktzbOkZ3yh5UGWC28dFMCr/ykF0+P64a/j5nCYjtHykyJFRbbeWPlfldgqorcwmLuf+8HzucWAXAiXSu+RKTxUdgRaUBahfrxnwcGc/MVrbEb8OqyfTg3OT+dVcDprAJMJugWHVTpZ3h5mehacn1PSpbr9UVbj/PGygM8/p8fq9QXwzB46r872JuaRaDVAsDx9DzsVVgxJiLSkHg07Kxdu5aJEycSGxuLyWTi008/dbtuGAYzZ84kNjYWPz8/RowYwa5du9zaFBQUMH36dCIiIggICGDSpEkcP66hdmm8fL3NvHhTPD4WL05nFXD4jGN0Zk/JqE77iAD8fSwX/QxnGNqXWhp2thxNd31O0pnKi6Cdlu1K5fMdKVi8TLx1d3/MXiYKbXbSsgqqdV8iIp7i0bCTk5ND7969mTt3boXXX331VV5//XXmzp3L5s2biY6OZvTo0WRllf4DPmPGDBYvXszChQtZv3492dnZTJgwAZtNS3Cl8fL1NnNFm1AAvj98Diidwipfr1ORrlGOsLM3tXTKalvyedevq7Jj84ZDZwHHxoWDO4YTE+ILUGnxtIhIQ+XRsDN+/Hj++Mc/Mnny5AuuGYbBG2+8wbPPPsvkyZOJj4/n3XffJTc3lw8//BCAjIwM3n77bV577TVGjRpF3759WbBgAYmJiaxcubK+b0ekVg3qEA7AxsOO0OGstelRhbDTraTN3pKRnYy8Ig6mle7QXJWws+N4BgBXtG0BQOsWfgAkK+yISCPTYGt2kpKSSE1NZcyYMa7XrFYrw4cPZ8OGDQBs2bKFoqIitzaxsbHEx8e72lSkoKCAzMxMt4dIQzOwfWnYMQyj0uLkijinsY6n55GVX8SO4+cBiAi0YvYysetk5kX38ykstru+r3frUADiWvgDuJ3JJSLSGDTYsJOamgpAVFSU2+tRUVGua6mpqfj4+NCiRYtK21Rk1qxZhISEuB5xcXG13HuRmuvbJhQfixdpWQXsScnicMnZWT2rMLIT6u9DdLBj2mn/qSy2HTsPwNBO4QzqEAbAl4mOvyM2u8HqfWmczy10vX//qSwKi+0E+1poG+4IOXFhjv/VNJaINDYNNuw4mUwmt+eGYVzwWnmXavPMM8+QkZHheiQnJ1faVsRTfL3N9I0LBeD9jUewGxAR6EPLIGuV3u9ckbU3NYttxxzFyX3iQrkuIQZwTGUVFNuY9uFWps7fzDOLEl3vdU5h9Wod6vq75JrG0siOiDQyDTbsREdHA1wwQpOWluYa7YmOjqawsJD09PRK21TEarUSHBzs9hBpiAaW1O0s2urYubh7TPAlw75Ttxjn8vNMtpcUJ/dt04KxPaPxMkHiiQxuf+t7vtrp+Du2am8auYXFAK5pr4TWIa7Pc43snNfIjog0Lg027LRv357o6GhWrFjheq2wsJA1a9YwZMgQAPr164e3t7dbm5SUFHbu3OlqI9KYOaecCoodOyFXpTjZyVm38/WeNNJzi/CxeNEjJpiIQKurHmjL0XT8fcyEBfhQUGxn3QHHrs3OkZ3eZcNOSc3OyfP5tbIzs4hIffFo2MnOzmb79u1s374dcBQlb9++nWPHjmEymZgxYwYvvfQSixcvZufOnUydOhV/f39uv/12AEJCQrj33nt5/PHH+frrr9m2bRt33nknCQkJjBo1yoN3JlI7rmjTAh9z6V/TqhQnO3WNcrRNycgHID42GB+L47Mm9o4FIMTPmw/uG8gNfRzPV+w+RX6RjX2nHKu4EkqKkwEig6z4mL2w2Q3XZ4qINAYX35msjv3www+MHDnS9fyxxx4D4J577uGdd97hqaeeIi8vjwcffJD09HQGDhzI8uXLCQoq3T129uzZWCwWpkyZQl5eHtdeey3vvPMOZrO53u9HpLb5epvpExfKpiOOvXYuZ2SnY2QAFi8TxSU7HveJKy3kn9K/NeAoWG4bHkBekY353x5h1d40dp7IwGY3iAj0IbZkbx1w7MzcqoUfSWdyOJ6e55rWEhFp6DwadkaMGOHaCr8iJpOJmTNnMnPmzErb+Pr6MmfOHObMmVMHPRTxvEEdwth05Bw+Fi/aRwRU+X1Wi5kOLQPYf8qxiqtvySaF4Dgp/faBbVzPB7QLI8TPm3M5hczfcARwL052al0SdpLTcxlMePVvSkSkHjXYmh0RcbimexQmEwxsH4bFfHl/ZbtFl44ElQ075XmbvRjZtSUAX+xwbDiY0CrkgnatS+p2jp+7eJHyd4fO0vv55Xy8+dhl9VdEpC4o7Ig0cH3iQvls2jDeuKXPZb/Xufw8ItBKq1C/i7Yd3SPa7XnvuAvDTlyY4zOOX+L08w++P0pGXhF//HwP53IKL9q2Oopsdr47dBabDiUVkSpQ2BFpBOJbhRAeWLX9dcoa0bUlFi8TE3rFXHLJ+vCuLd2KoRNahV7QxrWL8kU2FrTZDdYfdKzqyioo5i9fH3C7nlNQXNXuV+r3S3Zx21sbef+7IzX+LBFp+hR2RJqwnrEhbP3daJ67vvsl2wZaLQzu6KjDiQ3xrXDzwqpsLLjzRAbnc4vwNjvC1YKNRzl8Optim53f/28nPX+/jHdL6oKq48T5PP692bER6Nd706r9OSLSfCjsiDRxwb7eVa71mdDLsbuy8xDS8pwrsE5l5VNQbKuwzdr9pwG4plsk13SLpNhu8MLnu5k6fzPvfncUgH+tP4y9mlNQ/1hzyLXC7Icj6RQWa88fEbk4j67GEpGG5Sf9WtMyyEqfkmMqygsP8MHP20xekY2T5/MrXB229oAj7FzdpSVXtgtj9b40Vu9zvObv49gSIvlcHhsPn2VIp4jL6l9aVj4LS0Z1vM0m8ops7Dh+nv7twi7rc0SkedHIjoi4mEwmRnSNJNTfp9Lrzqms4+m5GIZB8rlc1xYSmflFbC05dPTqzi3pHBXEbVc6lrjHhvjy3weGcGPfVgD8+4fLP5Pu7XVJFBbb6dsmlNE9HEfCfHfo7GV/jog0Lwo7InJZnFNZy3edYso/vuOqV7/hif/swDAMNhx0rJDqEBHgavf7iT356+1X8PnDV9EjNpgp/eMA+GpnKhl5RVX+3vScQt7f6JgGm35NJwaXTLV9d1hhR0QuTtNYInJZ4kpGdpzBA+CTrcfp364FiSccZ2pd3aWl65qPxYvrS2qBwHHeVpeoQPafyuazH09y56C2Vfrev689RG6hjR4xwYzsGsmh047NErccTaeg2IbVol3TRaRiGtkRkcvSoWUgACaTo8bnVyM6Ao7l4MtKTlC/qnPltTgmk8k1uuOcyjp5Po9lu1IrLXreeSKDf61LAuCx0V0wmUx0bBlIRKCVgmI720umzkREKqKRHRG5LD/p1xqb3WBQh3B6xAZjtxvsTcnkm32nOVtciLfZVOlqLqeb+rbilaV72XE8gxv/+i3bk88DMHVIO2ZO6unWtshm56n/7sBmN7g+IYZRJbU6JpOJQR3C+HxHCt8dPsvAS3yniDRfGtkRkcsSYLXw82HtXSewe3mZeG1KH2JKDg3t3zaMAOvF/zsqPNDKqO6O0OIMOgALNx8jvdyOy/9Yc4jdKZmE+ntfEIScoUpFyiJyMQo7IlJjYQE+/P3OflzZPowHR3as0nueHteNib1j+c113djw62voERNMfpGdDzeVnqd14FQWf/n6IAC/n9jjgo0OnZsgbks+T35RxVNgIiIKOyJSK3rHhfLvXw7mqs4tL90YaBcRwJzb+vKLqzsSG+rH/Ve3B+CdDUcoKLaRW1jMtA+3UWizM7JrS27s0+qCz+gQEUBkkJXCYjs/HEmv1fsRkaZDYUdEGoQJvWKJDvbldFYBS7af5JlFiew7lUXLICuv3NyrwrO9TCYTQ0s2JvzVB1v48Ptj1d6ZWUSaLoUdEWkQvM1eTB3aDoDnP9vN/7afxOxl4q+3X0FksG+l73tybFd6tw4hK7+Y3yxO5NZ/buTLxBS3A0dtdoO8Qk1ziTRXJsO59WkzlpmZSUhICBkZGQQHB3u6OyLNVkZuEYNf/prckmDy3PXdue+qDpd8n81u8M6GI7y2fJ/rvVaLF33iQjmTXUDyuTyK7Hb+dXd/ri0pjBaRxq+qP7+19FxEGowQf2/uGNiGt9YlcX2vGO4d1r5K7zN7mbh3WHvG9ozi3Q1HWLbrFMfO5fJ90jm3drO+2suIrpGYvS6cEquJ01kFfHvwDF5eJnzMXrQK9SOhdUitfoeIVJ9GdtDIjkhDUlhs5/ukswzqEI53FU9rL88wDPamZpF4IoOYEF+ig3356T++43xuEbNv6c1NfVvXap+n/P07Nh1xD1b/eWAwA3RAqUidqurPb9XsiEiD4mPx4qrOLasddMBRuNw9xnEO11UlB5LeXzId9ubKAxTb7Bd9/+p9aTz04VaSz+Ve8rt2n8xk05FzWLxMDO4Q7tpvaNXetGr3X0Rql8KOiDQLU4e0IzzAhyNnc1m09USl7ex2g+c+3ckXO1K4993NZOVf/LDSDzc5zggb2zOaj34xiCfGdAVgow4oFWkwFHZEpFkIsFp4YLhjw8M3vz5AYXHFoztrD5zmeHoeAPtPZTNj4XZslSxnzy4oZnFJcLpjYBsABnZwTF3tOJ7htiJMRDxHYUdEmo07B7WlZZCVE+fzeO7TxAqnsz783rGD81WdI7BavPh6bxqvLt1b4ect2X6SnEIbHSICXLs5t27hT+sWftjsBj8c1UaHIg2Bwo6INBt+PmZ+N6EHXib49w/HeWDBFrf9d1Iy8vi6pNbm9xN78Kef9gbgH2sP8/i/f+RcmXO7DMPgg+8dU1i3D2zjtumh88yu76swlbVsVyqvL99X6eiRiNScwo6INCsTe8cy785+WC1erNyTxh3/2sjprAIAPt6cjM1ucGX7MDpFBjGpdyxPjeuKyQSfbD3Ota+tZsHGo+w8kcHq/afZdTITH4sXN1/hvrprYHvHVNbF6nbsdoNXl+7ll+9v4S+rDrL+4Jm6u2mRZk5hR0SanbE9o1lw30CCfS1sPXae8W+u45t9aSzclAyU1t8APDiiE/99YAjdooNIzy3iuU93MmHOen42fzMAExJiaBHg4/b5zpGdyup2cguLefCDrfxt9SHXa3tTMmv9PkXEQWFHRJqlAe3CWPTgULpGBXEmu4Cfzd9MamY+YQE+jIuPdmvbr20LPps+jGfGdyO+VTCRQVbMXib8fcwV7vAcF+ZPq1A/iu0GWyqo23n6k0SW7krFx+zFlSV78ew7lVU3Nyoi2kFZRJqvTpGB/G/aUF78Yg/vb3TU3/y0f2usFvMFbb3NXvxyeEd+WbKiy2Y3sBtGpfsBDewQxqKtJ9h4+CxXdyk9Cd5mN/h6zykA/nl3PwqK7Ww6co79CjsidUZhR0SaNV9vM3+4MZ6R3Vqy4eBZHhrZqUrvM3uZMFP5sRODOoSzaOuJC46sOHQ6m9xCG/4+Zq7q3NK1ceGBU9nY7MZlHWXx8eZjHDqdw4xRnfH30T/nIpXR3w4REeCablFc0632DgkdXFK382PyeXILi11hZHvyeQASWoVg9jIRF+aPr7cX+UV2jp3LpX1EwCU/2zAMXlu+n7nfHARg69F0/u9nAwj29a61/os0JarZERGpA61b+BEb4kux3WDzkdK6nR3HzwPQOy4UcIwQdY4MAqjSVJZhGMz6aq8r6Ph5m/nhaDp3vPW929J4ESmlsCMiUgdMJhNDO0UAsGbfadfrPyZnANC7dajrtS5RJWEntTTsvLX2MPe9u5mMXPfjKl5eupd/rj0MwPOTevKfBwYTFuBD4okMbv3ndxes/iostpNfZEOkOVPYERGpIyO7RQKwer9jo8L8Iht7SpaY92od4mrXJSoQKF2RlVtYzJ+X72PlnjT+vHyfq922Y+n8Y40j6Lx0UwL3DGlHfKsQ/v3LQUQGWdl/Kts14gOQV2jjxr9+y9CXV5GWlV+HdyrSsCnsiIjUkaGdIjB7mTh8Oofkc7nsScmk2G4QHuBD6xZ+rnZdot2nsdYdOENBydldH3x/lN0nM7HZDX73v10A/KRfa24vsxdQp8ggXrwpAYB/rTtM0pkcAF5ZupfdKZmczSlk/rdH6vx+RRoqhR0RkToS4udNvzYtAFi9L40fS4qTe8eFuh0v0bVkGuvw6RwKi+0s3+VYmu5j9sJuwMwlu1i4+RiJJzIIslp4ely3C75rVPdIhndpSZHN4IXPdvHtwTO8s+GI6/qCjUddJ7gX2ew88P4Wbvzrt67VYCJNmcKOiEgdGt7VscfO6n2n2XHcUa9TdgoLICbElyCrhWK7wYG0LFbtdYSdl29OwNfbi01HzjFziWNU59HRXWgZZL3ge0wmE7+b2ANvs4lv9p3mgQVbAMe5XR1bBpCVX8xHmxyHnP71m4Ms3ZXK9uTz3PrPjTUKPOdzCzmfq8JoadgUdkRE6tDIro66nQ2HzrL5qGPPHedKLCeTyeSaylq4KZn03CJC/b2Z1DuWh0Y49v0pshl0jQri7sFtK/2uji0D+fnQ9gBk5RfTNtyf567vzi+vdmyE+Pb6JDYfOcecVY66nvAAH06cz+PWf27k2NnLDzzZBcWMnr2W0bPXkq6VYNKAKeyIiNSh7jFBRAVbySuykXwuD3BfieXkXJH18Q+O87mu6RaJxezF/Vd3oH1EAF4meP6Gnlgq2bHZafq1nYkJ8cXiZeL1Kb3x97FwQ99YooKtnMos4O63N2GzG0zsHcuXj1xFh5YBnDifx6jX13DNn1dz9/9tYv63SRjGpU9hX7ozldNZBZzOKnArjBZpaBR2RETqkMlkYniZ4yLiwvwIK3dwKEDXkhVZhSWFyWN6ODY49PU288mvhrD80atdB4xeTKDVwv+mDWX5o1fTr63j3C2rxewa8ckrshEd7MsfbuhJVLAvC+8fRI+YYAptdg6fyWHt/tM8/9lut3qfIpudmUt2MXPJLrcQ9MmW465fv/fdEdX/SIOlsCMiUsecU1lQ8agOlK7IAvCxeHFV59KAFBbgQ6fIoIreVqHIIF86tAx0e+22gW0I8XPssPynn/Yi1N8RuCKDffl8+jDWPTWSD+8fyC+udhxs+uIXe9h6LJ1im50ZC7fzzoYjvLPhCF/tTAXgeHou3x0+Czh2gy6yGby6bB/lGYbBuxuOuAUjALvd4PUV+3m3TKgSqSs6LkJEpI4N7RyBxctEsd2oPOxElYaZYZ0iCLDW7j/Pwb7e/PeBwWTmF7lGfJy8So6tiAvzZ3CHcE6k5/FFYgrTPthK37Yt+CIxxdX2z8v2MaZHFJ9uOwE4jsV4bkJ3JsxZz2c/nuS+Ye3dapK2HE3n90t2YTLBFW1buI7DWHPgNH/5+gDgqDUa1jmiVu9XpCyN7IiI1LFgX2/G9IzC22xyrc4qLyLQSnjJ9JZzCqu2dY4KuiDolGcymXj55gTaRwRwMiOfL3akYPEyMfuW3oQF+HD4TA7/2XKcT7Y6ws7N/VrTMzaEm/q2AuClL/e4TXW9+53jNHnDwG0Up+y+P79ZnEheYfV2ebbbDd7/7ggbS0aZRCqisCMiUg9en9KHb399jdsITnnTr+nEqO6RTOgdW489u1CQrzd/u+MKfL29MHuZ+Mttfbmpb2seHOFY1fXHz3eTdCYHP28z4+KjAXhiTFesFi++TzrHsl2Oqa5Tmfl8VWZU6N8/JJORV8TBtGzW7j+NyeQIecfO5fJmySjP5Xr3uyP89n+7uP+9H6odmKTpa9BhZ+bMmZhMJrdHdHS067phGMycOZPY2Fj8/PwYMWIEu3bt8mCPRUQq5uttJjLI96Jtpg5tz7/uGUBgLU9hVUf3mGCWzxjOshlXc11CDAB3DmpLbIgvOSWhYnx8tKuvsaF+/LKk3uePX+whv8jGh98fo9hu0L9tC7pEBZJbaOM/PyTzzoYkAEZ1j2LWZMfOz2+tO8yukxkV9qXYZmdT0jleX76PL8uEp6Nnc3hl6V7AsdT+q50pFb5fpEGHHYCePXuSkpLieiQmJrquvfrqq7z++uvMnTuXzZs3Ex0dzejRo8nKuvTJwSIicnFtwv3pFFla6OzrbWbG6C6u5zf3a+3W/oERHYkO9uV4eh7zVh/iw5JNDKcObedaDfZ/65P4ZItjCuxnQ9sxukcU1yVEY7MbPP7vHzmbXeD6vMz8In79yQ76/XElU/7xHX9ZdZAHP9jKrK/2UGyz89R/d5BfZMfX2/GjbOHm5Fq79+8OnSUtU+eJNRUNPuxYLBaio6Ndj5YtHfPdhmHwxhtv8OyzzzJ58mTi4+N59913yc3N5cMPP/Rwr0VEmqbJfVsxrmc04+OjGVxuKby/j4VnrnMcZfHm1wc4nVVAVLCVsT2jubFvK1r4e3MyI5+8Ihtdo4Jc7585sSfhAT7sTc3iJ3//juRzuRxMy+bGud+ycLNj6ivU35urSoqY/7HmMOPfXMf3Sefw8zbz3s8H4mWCTUnnOHw6u8b3uP7AGW57ayPXvr6GJT+erPHniec1+LBz4MABYmNjad++PbfeeiuHDztO/E1KSiI1NZUxY8a42lqtVoYPH86GDRsu+pkFBQVkZma6PURE5NIsZi/+flc/5t3ZDy8v0wXXJ/WOpV/bFq7ndwxsi7fZC19vs9vhpT8b2s51PlhksC//fmAwrUL9SDqTw01/28CNf/2Ww2dyiAnxZcG9A9ny3Gjev3cgb9zSBx+zFwfSHKHm6XFdubJ9GCNKlvc7N2Us73xuIaezCrDZL71Z4mclAScrv5iHP9rG4//+keyC4ir+Dl1aVn4Rry7dy75UzULUlwYddgYOHMh7773HsmXLeOutt0hNTWXIkCGcPXuW1FRHAVxUlPuqhaioKNe1ysyaNYuQkBDXIy4urs7uQUSkOTGZTPx+Yg9MJsd+QbdeWfrv612D2hFktRAT4suNJau3nDq2DGTRg0PoFh3EmewCsguKubJ9GJ9NH8awzo7T4wFu7NuKD+8fSJswf65LiObuwe0AuGWA43s+2XLctTGj05aj5xj40tcMeHElnZ/9kv5/XMGIP33D2NlruWHuev5dZvqr2GZnxR7H2WTXJUTjZYJPth7nwQ+2VmlX6ap4Y+UB/rb6EC9/teeCa3a7UWvfI6VMRiP6Xc3JyaFjx4489dRTDBo0iKFDh3Ly5EliYmJcbe6//36Sk5NZunRppZ9TUFBAQUGZeeHMTOLi4sjIyCA4OLhO70FEpDn47tBZfCxebqM8ACkZeXibvYgIvPAwU3DU6bz0xR5aBll5+NrOeFdyPIbzR5dzdKjIZmfIy6s4nVXA3++8gnHxjp8L53IKuf4v60jJqLz+xs/bzMZnriXE35vvDp3ltrc20sLfm83PjmLL0XTuensThTY77/38Sq7uUvHWAVWVmV/EkFmryC4oJjzAhx+eG+W6h01J57jtrY08M74b913VoUbf01xkZmYSEhJyyZ/fDXpkp7yAgAASEhI4cOCAa1VW+VGctLS0C0Z7yrNarQQHB7s9RESk9gzuGH5B0AGICfGrNOiAY0+il2/uxeNjulYadADXCl0nb7MXPykpmH7z64McOp2N3W7w2L+3k5KRT4eIAH78/Rg2PXstXz58FZ/8ajAf3DeQLlGB5BXZWLjZUUztXDY/qnsUFrMXAzuEc+cgx+Grryzdi70K02AX8/GmZNeU2NmcQk5llv6H92c/nsRmN3h7fVK1vicrv6hGfWvKGlXYKSgoYM+ePcTExNC+fXuio6NZsWKF63phYSFr1qxhyJAhHuyliIh4wu1XtiHAx8yelEzGvbGWu/7ve1bvO43V4sVf77iCED9vIoN86REbTL+2YQztFOEaQXl3wxEKi+0sLTkOw7l/EMC0azoRaLWw62Qmn+24vILlrPwi1yhUsc3O/G8dy+6dOW3nidLl9tuS0wFIychnW/L5y/qe/20/QcLM5by9Pumy3tdcNOiw88QTT7BmzRqSkpL4/vvv+clPfkJmZib33HMPJpOJGTNm8NJLL7F48WJ27tzJ1KlT8ff35/bbb/d010VEpJ7Fhfnz+cNXMbJrS4psBt8edOyq/MINPekeU/EI/qTesUQE+nAyI59Xl+4lNTOfAB8zQzuVHl8RFuDj2kPoteX7L6gJqsyGQ2e44g8ruOlvG0g+l8uXO1M5mZFPRKCPa++inSV7C+UV2tibUlqw/MWOy9szyBly3ly5XyM8FWjQYef48ePcdtttdO3alcmTJ+Pj48PGjRtp29YxpPjUU08xY8YMHnzwQfr378+JEydYvnw5QUFVPzBPRESajvYRAcz/2ZX839T+XNEmlAeGd2RK/8oXofh6m7ljoONnyr9KAsPIbpH4epvd2t17VXvXbs+zvtrDmTL7AVXmn2sPU2Qz2J58nuveXMefljk2QLxrUDv6tXFM8e084VgNvPNkBsVlpq6+TEyp8lTWgVNZ7DjuCE2Z+cW8v/Fold7XnHh+m86LWLhw4UWvm0wmZs6cycyZM+unQyIi0ihc0y2Ka7pV7YyxOwe1Zd7qQxTaHCM2ZaewnPx9LMwY1ZnnPt3J/G+P8N53RxnepSWjukcxsEMYHSIC3GqITpzPY83+0wDEtwpm54lMsgqKsVq8uHNQGw6dzgFw7Rq97ZhjCmtE15ZsOZJOamY+25LTL3mWGcCikkNZW/h7k55bxL/WJTF1SDv8fRr0j/h61aBHdkREROpayyArN/RxnEfmY/Fy7dlT3h0D2/Dy5AR6x4Visxus2pvGbxYncu1raxjw4td8suW4q+2/NydjGI5T4Rc/OJSHRnbE22zil8M7Eh5opUesY1otJSOfM9kFbC+p0RnUIZzRJQfBfl6FqSy73XCdQP/8DfG0CfPnXE4hH35/rNq/H02Rwo6IiDR7D4zoSKi/N7cOiKv0bDKTycStV7bhfw8NZeVjw5kxqjMD24fhY/HiTHYBv160g50nMrDZDf5TsrnhrVfG4W324smx3dj9wjgeKzluI9BqoUNEAAC7Tmay7dh5APrEhbrqeaoylbXx8FlSMvIJ9rUwpkeU67DWf649TH6R+8Gop7MKeHPlAdKymt8xGAo7IiLS7HVsGci2347mhRviq9S+U2QgM0Z14eNfDmbH78cwtmcURTaDhxduY9kuRyFyqL83Y3uWTomVX0rfs1UIAF/vOUVKRj5eJujVOoSrukQQZLVwKrOArSXTW2WdzipwhaBPtjpGdSb0jsXX28zkK1oTG+JLWlYBCze5j+48s2gHs1fu55GPtje7jQsVdkRERMCt5uZy+HqbeXlyLyKDrBw+ncOjH28HYHLf1hcUOpeV0MoxlbWoJLB0iw7G38eC1WJ2TWW98Pluks446nvO5RTy8EfbGPDiSka+tpq5qw64TnqfXLIjtY/Fi1+N7ATAX1YdJCPPsTJre/J5Vu5JA+C7w2f5b5kpt+ZAYUdERKSGWgT48NqU3gAUlCxNv+3Kix9FFB/rGNlxbjLYp02o69rPh7UnwMfMjuMZjH9zLS99uYcxs0sPJj16Npc/L99PbqGNNmH+bhs43jogjk6RgZzLKWTuqgMAvL5iPwCRQY4NHV/8co/bCfNO53IK+b/1SSzdmULSmZwqnSVWbLPzw5FzvLp0Lw99uNUVzhoSlWqLiIjUgqs6t+TeYe15e30SA9q1oHPUxbdB6VkSdpz6xoW6fh3fKoRlj17NU//dwYZDZ/nnWsch2F2iAvnjjQkcO5fLx5uPsflIOr8c3uGC3aSfu747U+dv5p0NR+gcFcTa/aexeJlY+ItBPPjBVvamZvHiF3t4/ZY+bn147N/bWb3vtOt5gI+ZG/u24ufD2tOxZaBb26Nnc5j/7REWbzvhGkEC2Ho0nX//cjBxYf5V+n2rD43qbKy6UtWzNURERC6myGbn8x0nGdg+nNhQv0u2v+rVVSSfywNg5WPD6RTpHijsdoMF3x9l/rdHuC4hmoev7YzVUjo1VmyzY6nkWI2p8ze5BZfbrmzDrMkJbDuWzuR5GzAMePfnVzK85LyvNftPc8//bcLbbKJrdBAHTmW7RqkAruocQatQP3y9zRxPz+PrvadwJogQP2+Gd2nJzpMZHD6dQ9twf/79y8FEBftW7Teumqr681thB4UdERHxjF8t2MJXO1MJ9rWw/Xdj8PKqXt1QRQ6mZTH2jXXY7AY+Zi9WPznCFcBmLtnFOxuOEOrvzZKHhhEb6st1f1nH/lPZ3DusPb+d0AOb3eD7pLP83/ojbsGmrOFdWvLzYe0Z2jEci9mL1Ix8fvqPDSSfy6NzZCAf/3IwYQE+tXZP5VX157emsURERDwkvlUIX+1MpU+bFrUadAA6RQZxz+B2/N+3Sdw1uK3bSNOvx3djW/J5fkw+z33vbean/eLYfyqbUH9vHr6mMwBmLxNDOkYwpGMESWdy+GZvGrmFxRQU2zGZTEzoFUOXclN10SG+fHjfIH769+84kJbNL977gQX3DbxooXZ90MgOGtkRERHPyMgr4pWle7n9yjbEtwq59Bsuk81usPnIOfq3bXHBdNepzHwmzllPWlZpofLvJ/bgZ0Pb1/h7D5zKYvK8DWTlFzOpdyxv3tqn2qvdLqaqP7+1GktERMRDQvy8eemmhDoJOuAYnRnUIbzCup6oYF/+eXd/fCyOax0iArhzUNta+d7OUUH8/c5+WLxMLPnxJLNLVoN5isKOiIhIM9UnLpQ3b+lD58hAXpqccMHGhzUxtFMEL92UADj2/PHk3j6q2REREWnGxifEML7kiIraNmVAHEfP5fCvdUkE+HiubkdhR0REROrM46O7MvmK1hfs01OfNI0lIiIidcbLy+TRoAMKOyIiItLEKeyIiIhIk6awIyIiIk2awo6IiIg0aQo7IiIi0qQp7IiIiEiTprAjIiIiTZrCjoiIiDRpCjsiIiLSpCnsiIiISJOmsCMiIiJNmsKOiIiINGkKOyIiItKkWTzdgYbAMAwAMjMzPdwTERERqSrnz23nz/HKKOwAWVlZAMTFxXm4JyIiInK5srKyCAkJqfS6ybhUHGoG7HY7J0+eJCgoCJPJ5Onu1KrMzEzi4uJITk4mODjY092pc7rfpqs53Svofpu65nS/dXmvhmGQlZVFbGwsXl6VV+ZoZAfw8vKidevWnu5GnQoODm7yf6HK0v02Xc3pXkH329Q1p/utq3u92IiOkwqURUREpElT2BEREZEmTWGnibNarfz+97/HarV6uiv1QvfbdDWnewXdb1PXnO63IdyrCpRFRESkSdPIjoiIiDRpCjsiIiLSpCnsiIiISJOmsCMiIiJNmsJOEzBr1iwGDBhAUFAQkZGR3Hjjjezbt8+tjWEYzJw5k9jYWPz8/BgxYgS7du3yUI9r16xZszCZTMyYMcP1WlO73xMnTnDnnXcSHh6Ov78/ffr0YcuWLa7rTel+i4uLee6552jfvj1+fn506NCBF154Abvd7mrTWO937dq1TJw4kdjYWEwmE59++qnb9arcV0FBAdOnTyciIoKAgAAmTZrE8ePH6/Euqu5i91tUVMTTTz9NQkICAQEBxMbGcvfdd3Py5Em3z2gq91veL3/5S0wmE2+88Ybb603tfvfs2cOkSZMICQkhKCiIQYMGcezYMdf1+rpfhZ0mYM2aNTz00ENs3LiRFStWUFxczJgxY8jJyXG1efXVV3n99deZO3cumzdvJjo6mtGjR7vOBWusNm/ezD//+U969erl9npTut/09HSGDh2Kt7c3X331Fbt37+a1114jNDTU1aYp3e8rr7zC3//+d+bOncuePXt49dVX+dOf/sScOXNcbRrr/ebk5NC7d2/mzp1b4fWq3NeMGTNYvHgxCxcuZP369WRnZzNhwgRsNlt93UaVXex+c3Nz2bp1K7/97W/ZunUrixYtYv/+/UyaNMmtXVO537I+/fRTvv/+e2JjYy+41pTu99ChQwwbNoxu3bqxevVqfvzxR37729/i6+vralNv92tIk5OWlmYAxpo1awzDMAy73W5ER0cbL7/8sqtNfn6+ERISYvz973/3VDdrLCsry+jcubOxYsUKY/jw4cYjjzxiGEbTu9+nn37aGDZsWKXXm9r9Xn/99cbPf/5zt9cmT55s3HnnnYZhNJ37BYzFixe7nlflvs6fP294e3sbCxcudLU5ceKE4eXlZSxdurTe+l4d5e+3Ips2bTIA4+jRo4ZhNM37PX78uNGqVStj586dRtu2bY3Zs2e7rjW1+73llltcf28rUp/3q5GdJigjIwOAsLAwAJKSkkhNTWXMmDGuNlarleHDh7NhwwaP9LE2PPTQQ1x//fWMGjXK7fWmdr9Lliyhf//+/PSnPyUyMpK+ffvy1ltvua43tfsdNmwYX3/9Nfv37wfgxx9/ZP369Vx33XVA07tfp6rc15YtWygqKnJrExsbS3x8fKO+d6eMjAxMJpNr1LKp3a/dbueuu+7iySefpGfPnhdcb0r3a7fb+eKLL+jSpQtjx44lMjKSgQMHuk111ef9Kuw0MYZh8NhjjzFs2DDi4+MBSE1NBSAqKsqtbVRUlOtaY7Nw4UK2bt3KrFmzLrjW1O738OHDzJs3j86dO7Ns2TIeeOABHn74Yd577z2g6d3v008/zW233Ua3bt3w9vamb9++zJgxg9tuuw1oevfrVJX7Sk1NxcfHhxYtWlTaprHKz8/n17/+NbfffrvrsMimdr+vvPIKFouFhx9+uMLrTel+09LSyM7O5uWXX2bcuHEsX76cm266icmTJ7NmzRqgfu9Xp543MdOmTWPHjh2sX7/+gmsmk8ntuWEYF7zWGCQnJ/PII4+wfPlyt7nf8prK/drtdvr3789LL70EQN++fdm1axfz5s3j7rvvdrVrKvf78ccfs2DBAj788EN69uzJ9u3bmTFjBrGxsdxzzz2udk3lfsurzn019nsvKiri1ltvxW6387e//e2S7Rvj/W7ZsoU333yTrVu3XnbfG+P9OhcU3HDDDTz66KMA9OnThw0bNvD3v/+d4cOHV/reurhfjew0IdOnT2fJkiV88803tG7d2vV6dHQ0wAVJOS0t7YL/imwMtmzZQlpaGv369cNisWCxWFizZg1/+ctfsFgsrntqKvcbExNDjx493F7r3r27a0VDU/vzffLJJ/n1r3/NrbfeSkJCAnfddRePPvqoaxSvqd2vU1XuKzo6msLCQtLT0ytt09gUFRUxZcoUkpKSWLFihWtUB5rW/a5bt460tDTatGnj+nfr6NGjPP7447Rr1w5oWvcbERGBxWK55L9d9XW/CjtNgGEYTJs2jUWLFrFq1Srat2/vdr19+/ZER0ezYsUK12uFhYWsWbOGIUOG1Hd3a+zaa68lMTGR7du3ux79+/fnjjvuYPv27XTo0KFJ3e/QoUMv2Epg//79tG3bFmh6f765ubl4ebn/02Q2m13/pdjU7tepKvfVr18/vL293dqkpKSwc+fORnnvzqBz4MABVq5cSXh4uNv1pnS/d911Fzt27HD7dys2NpYnn3ySZcuWAU3rfn18fBgwYMBF/+2q1/ut1XJn8Yhf/epXRkhIiLF69WojJSXF9cjNzXW1efnll42QkBBj0aJFRmJionHbbbcZMTExRmZmpgd7XnvKrsYyjKZ1v5s2bTIsFovx4osvGgcOHDA++OADw9/f31iwYIGrTVO633vuucdo1aqV8fnnnxtJSUnGokWLjIiICOOpp55ytWms95uVlWVs27bN2LZtmwEYr7/+urFt2zbX6qOq3NcDDzxgtG7d2li5cqWxdetW45prrjF69+5tFBcXe+q2KnWx+y0qKjImTZpktG7d2ti+fbvbv10FBQWuz2gq91uR8quxDKNp3e+iRYsMb29v45///Kdx4MABY86cOYbZbDbWrVvn+oz6ul+FnSYAqPAxf/58Vxu73W78/ve/N6Kjow2r1WpcffXVRmJiouc6XcvKh52mdr+fffaZER8fb1itVqNbt27GP//5T7frTel+MzMzjUceecRo06aN4evra3To0MF49tln3X4ANtb7/eabbyr8u3rPPfcYhlG1+8rLyzOmTZtmhIWFGX5+fsaECROMY8eOeeBuLu1i95uUlFTpv13ffPON6zOayv1WpKKw09Tu9+233zY6depk+Pr6Gr179zY+/fRTt8+or/s1GYZh1O5YkYiIiEjDoZodERERadIUdkRERKRJU9gRERGRJk1hR0RERJo0hR0RERFp0hR2REREpElT2BEREZEmTWFHREREmjSFHRHxiHbt2vHGG29Uuf3q1asxmUycP3++zvokIk2TdlAWkSoZMWIEffr0uayAcjGnT58mICAAf3//KrUvLCzk3LlzREVFYTKZaqUPl2v16tWMHDmS9PR0QkNDPdIHEbl8Fk93QESaDsMwsNlsWCyX/qelZcuWl/XZPj4+REdHV7drItKMaRpLRC5p6tSprFmzhjfffBOTyYTJZOLIkSOuqaVly5bRv39/rFYr69at49ChQ9xwww1ERUURGBjIgAEDWLlypdtnlp/GMplM/Otf/+Kmm27C39+fzp07s2TJEtf18tNY77zzDqGhoSxbtozu3bsTGBjIuHHjSElJcb2nuLiYhx9+mNDQUMLDw3n66ae55557uPHGGyu916NHjzJx4kRatGhBQEAAPXv25Msvv+TIkSOMHDkSgBYtWmAymZg6dSrgCHmvvvoqHTp0wM/Pj969e/Pf//73gr5/8cUX9O7dG19fXwYOHEhiYuIlv1dEak5hR0Qu6c0332Tw4MHcf//9pKSkkJKSQlxcnOv6U089xaxZs9izZw+9evUiOzub6667jpUrV7Jt2zbGjh3LxIkTOXbs2EW/5/nnn2fKlCns2LGD6667jjvuuINz585V2j43N5c///nPvP/++6xdu5Zjx47xxBNPuK6/8sorfPDBB8yfP59vv/2WzMxMPv3004v24aGHHqKgoIC1a9eSmJjIK6+8QmBgIHFxcXzyyScA7Nu3j5SUFN58800AnnvuOebPn8+8efPYtWsXjz76KHfeeSdr1qxx++wnn3ySP//5z2zevJnIyEgmTZpEUVHRRb9XRGpBrZ+jLiJN0vDhw41HHnnE7bVvvvnGAIxPP/30ku/v0aOHMWfOHNfztm3bGrNnz3Y9B4znnnvO9Tw7O9swmUzGV1995fZd6enphmEYxvz58w3AOHjwoOs9f/3rX42oqCjX86ioKONPf/qT63lxcbHRpk0b44Ybbqi0nwkJCcbMmTMrvFa+D85++vr6Ghs2bHBre++99xq33Xab2/sWLlzoun727FnDz8/P+Pjjjy/5vSJSM6rZEZEa69+/v9vznJwcnn/+eT7//HNOnjxJcXExeXl5lxzZ6dWrl+vXAQEBBAUFkZaWVml7f39/Onbs6HoeExPjap+RkcGpU6e48sorXdfNZjP9+vXDbrdX+pkPP/wwv/rVr1i+fDmjRo3i5ptvdutXebt37yY/P5/Ro0e7vV5YWEjfvn3dXhs8eLDr12FhYXTt2pU9e/ZU63tFpOo0jSUiNRYQEOD2/Mknn+STTz7hxRdfZN26dWzfvp2EhAQKCwsv+jne3t5uz00m00WDSUXtjXILTMuv3Cp/vbz77ruPw4cPc9ddd5GYmEj//v2ZM2dOpe2d/fviiy/Yvn2767F79263up3KOPt3ud8rIlWnsCMiVeLj44PNZqtS23Xr1jF16lRuuukmEhISiI6O5siRI3XbwXJCQkKIiopi06ZNrtdsNhvbtm275Hvj4uJ44IEHWLRoEY8//jhvvfUW4Pg9cH6OU48ePbBarRw7doxOnTq5PcrWNQFs3LjR9ev09HT2799Pt27dLvm9IlIzmsYSkSpp164d33//PUeOHCEwMJCwsLBK23bq1IlFixYxceJETCYTv/3tby86QlNXpk+fzqxZs+jUqRPdunVjzpw5pKenX3SfnhkzZjB+/Hi6dOlCeno6q1atonv37gC0bdsWk8nE559/znXXXYefnx9BQUE88cQTPProo9jtdoYNG0ZmZiYbNmwgMDCQe+65x/XZL7zwAuHh4URFRfHss88SERHhWhl2se8VkZrRyI6IVMkTTzyB2WymR48etGzZ8qL1N7Nnz6ZFixYMGTKEiRMnMnbsWK644op67K3D008/zW233cbdd9/N4MGDCQwMZOzYsfj6+lb6HpvNxkMPPUT37t0ZN24cXbt25W9/+xsArVq14vnnn+fXv/41UVFRTJs2DYA//OEP/O53v2PWrFl0796dsWPH8tlnn9G+fXu3z3755Zd55JFH6NevHykpKSxZssRttKiy7xWRmtEOyiLSbNjtdrp3786UKVP4wx/+UG/fq52XRTxL01gi0mQdPXqU5cuXM3z4cAoKCpg7dy5JSUncfvvtnu6aiNQjTWOJSJPl5eXFO++8w4ABAxg6dCiJiYmsXLlStTAizYymsURERKRJ08iOiIiINGkKOyIiItKkKeyIiIhIk6awIyIiIk2awo6IiIg0aQo7IiIi0qQp7IiIiEiTprAjIiIiTdr/AzcwdxlWiYsEAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 准备优化器和模型\n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_layers = 1\n",
    "dropout = 0\n",
    "n_tags = len(dataset.label2id)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "\n",
    "lstm_crf = LSTM_CRF(vocab_size, hidden_size, n_layers, dropout, n_tags)\n",
    "train(lstm_crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8a9138a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.5786226446556612, recall = 0.41368621840724845, f1 = 0.4824469933958985\n",
      "precision = 0.5081967213114754, recall = 0.3606858542559706, f1 = 0.42191977077363896\n"
     ]
    }
   ],
   "source": [
    "evaluate(train_X, train_Y, lstm_crf, batch_size)\n",
    "evaluate(test_X, test_Y, lstm_crf, batch_size)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
